You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2011/10/31 15:26:36 UTC
svn commit: r1195471 [1/5] - in /incubator/lcf/trunk: ./
framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/
framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/
framework/core/src/main/java/org/apach...
Author: kwright
Date: Mon Oct 31 14:26:33 2011
New Revision: 1195471
URL: http://svn.apache.org/viewvc?rev=1195471&view=rev
Log:
More work for CONNECTORS-284 - review all indexes, and update all queries to present criteria in index order.
Added:
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ClauseDescription.java (with props)
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java (with props)
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/MultiClause.java (with props)
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCheckClause.java (with props)
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/UnitaryClause.java (with props)
incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BaseDerby.java (with props)
incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BaseHSQLDB.java (with props)
incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BigCrawlDerbyTest.java (with props)
incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BigCrawlHSQLDBTest.java (with props)
Modified:
incubator/lcf/trunk/build.xml
incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/Base.java
incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseHSQLDB.java
incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BasePostgresql.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopDeleteDeps.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PrereqEventManager.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ScheduleManager.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryHistoryManager.java
incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/ThrottleSpecManager.java
incubator/lcf/trunk/tests/cmis/src/test/java/org/apache/manifoldcf/cmis_tests/Base.java
Modified: incubator/lcf/trunk/build.xml
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/build.xml?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/build.xml (original)
+++ incubator/lcf/trunk/build.xml Mon Oct 31 14:26:33 2011
@@ -2301,7 +2301,7 @@
<target name="run-rss-loadtests-derby" depends="compile-rss-loadtests,calculate-rss-tests-condition" if="rss-tests.include">
<mkdir dir="test-output/rss"/>
- <junit fork="true" maxmemory="256m" dir="test-output/rss" outputtoformatters="true" showoutput="true" haltonfailure="true">
+ <junit fork="true" maxmemory="384m" dir="test-output/rss" outputtoformatters="true" showoutput="true" haltonfailure="true">
<classpath>
<fileset dir="framework/lib">
<include name="*.jar"/>
@@ -2328,13 +2328,15 @@
</classpath>
<formatter type="brief" usefile="false"/>
+ <test name="org.apache.manifoldcf.rss_loadtests.BigCrawlDerbyTest" todir="test-output/rss"/>
+
<!-- MHL -->
</junit>
</target>
<target name="run-rss-loadtests-postgresql" depends="compile-rss-loadtests,calculate-rss-tests-condition" if="rss-tests.include">
<mkdir dir="test-postgresql-output/rss"/>
- <junit fork="true" maxmemory="256m" dir="test-postgresql-output/rss" outputtoformatters="true" showoutput="true" haltonfailure="true">
+ <junit fork="true" maxmemory="384m" dir="test-postgresql-output/rss" outputtoformatters="true" showoutput="true" haltonfailure="true">
<classpath>
<fileset dir="framework/lib">
<include name="*.jar"/>
@@ -2369,7 +2371,7 @@
<target name="run-rss-loadtests-HSQLDB" depends="compile-rss-loadtests,calculate-rss-tests-condition" if="rss-tests.include">
<mkdir dir="test-HSQLDB-output/rss"/>
- <junit fork="true" maxmemory="256m" dir="test-HSQLDB-output/rss" outputtoformatters="true" showoutput="true" haltonfailure="true">
+ <junit fork="true" maxmemory="384m" dir="test-HSQLDB-output/rss" outputtoformatters="true" showoutput="true" haltonfailure="true">
<classpath>
<fileset dir="framework/lib">
<include name="*.jar"/>
@@ -2395,7 +2397,9 @@
<pathelement location="build/rss-loadtests/classes"/>
</classpath>
<formatter type="brief" usefile="false"/>
-
+
+ <test name="org.apache.manifoldcf.rss_loadtests.BigCrawlHSQLDBTest" todir="test-HSQLDB-output/rss"/>
+
<!-- MHL -->
</junit>
</target>
Modified: incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java (original)
+++ incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java Mon Oct 31 14:26:33 2011
@@ -126,8 +126,9 @@ public class IncrementalIngester extends
}
// Now, do indexes
- IndexDescription keyIndex = new IndexDescription(true,new String[]{outputConnNameField,docKeyField});
- IndexDescription uriHashIndex = new IndexDescription(false,new String[]{uriHashField});
+ IndexDescription keyIndex = new IndexDescription(true,new String[]{docKeyField,outputConnNameField});
+ IndexDescription uriHashIndex = new IndexDescription(false,new String[]{uriHashField,outputConnNameField});
+ IndexDescription outputConnIndex = new IndexDescription(false,new String[]{outputConnNameField});
// Get rid of indexes that shouldn't be there
Map indexes = getTableIndexes(null,null);
@@ -141,6 +142,8 @@ public class IncrementalIngester extends
keyIndex = null;
else if (uriHashIndex != null && id.equals(uriHashIndex))
uriHashIndex = null;
+ else if (outputConnIndex != null && id.equals(outputConnIndex))
+ outputConnIndex = null;
else if (indexName.indexOf("_pkey") == -1)
// This index shouldn't be here; drop it
performRemoveIndex(indexName);
@@ -153,6 +156,9 @@ public class IncrementalIngester extends
if (keyIndex != null)
performAddIndex(null,keyIndex);
+ if (outputConnIndex != null)
+ performAddIndex(null,outputConnIndex);
+
// All done; break out of loop
break;
}
@@ -385,18 +391,20 @@ public class IncrementalIngester extends
String oldURIHash = null;
String oldOutputVersion = null;
- // See what uri was used before for this doc, if any
- ArrayList list = new ArrayList();
- list.add(docKey);
- list.add(connection.getName());
while (true)
{
long sleepAmt = 0L;
try
{
+ // See what uri was used before for this doc, if any
+ ArrayList list = new ArrayList();
+ String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(docKeyField,docKey),
+ new UnitaryClause(outputConnNameField,connection.getName())});
+
IResultSet set = performQuery("SELECT "+docURIField+","+uriHashField+","+lastOutputVersionField+" FROM "+getTableName()+
- " WHERE "+docKeyField+"=? AND "+outputConnNameField+"=?",list,null,null);
+ " WHERE "+query,list,null,null);
if (set.getRowCount() > 0)
{
@@ -447,16 +455,17 @@ public class IncrementalIngester extends
try
{
+ ArrayList list = new ArrayList();
+
if (oldURI != null && (documentURI == null || !oldURI.equals(documentURI)))
{
// Delete all records from the database that match the old URI, except for THIS record.
list.clear();
- list.add(oldURIHash);
- // Ideally, we should include the actual string, but that's hard because of database limitations
- //list.add(oldURI);
+ String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(uriHashField,"=",oldURIHash),
+ new UnitaryClause(outputConnNameField,"=",connection.getName())});
list.add(docKey);
- list.add(connection.getName());
- performDelete("WHERE "+uriHashField+"=? AND "+ /* docURIField+"=? AND "+ */ docKeyField+"!=? AND "+outputConnNameField+"=?",list,null);
+ performDelete("WHERE "+query+" AND "+docKeyField+"!=?",list,null);
removeDocument(connection,oldURI,oldOutputVersion,activities);
}
@@ -464,12 +473,11 @@ public class IncrementalIngester extends
{
// Get rid of all records that match the NEW uri, except for this record.
list.clear();
- list.add(documentURIHash);
- // Database limitations prevent this
- // list.add(documentURI);
+ String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(uriHashField,"=",documentURIHash),
+ new UnitaryClause(outputConnNameField,"=",connection.getName())});
list.add(docKey);
- list.add(connection.getName());
- performDelete("WHERE "+uriHashField+"=? AND "+ /* docURIField+"=? AND "+ */ docKeyField+"!=? AND "+outputConnNameField+"=?",list,null);
+ performDelete("WHERE "+query+" AND "+ docKeyField+"!=?",list,null);
}
// Now, we know we are ready for the ingest.
@@ -524,8 +532,8 @@ public class IncrementalIngester extends
beginTransaction();
try
{
- int maxInClause = getMaxInClause();
-
+ int maxClauses;
+
HashMap docIDValues = new HashMap();
int j = 0;
while (j < identifierHashes.length)
@@ -541,51 +549,41 @@ public class IncrementalIngester extends
Iterator iter = docIDValues.keySet().iterator();
j = 0;
ArrayList list = new ArrayList();
- StringBuilder sb = new StringBuilder();
+ maxClauses = maxClausesRowIdsForDocIds(outputConnectionName);
while (iter.hasNext())
{
- if (j == maxInClause)
+ if (j == maxClauses)
{
- findRowIdsForDocIds(outputConnectionName,rowIDSet,list,sb.toString());
+ findRowIdsForDocIds(outputConnectionName,rowIDSet,list);
list.clear();
- sb.setLength(0);
j = 0;
}
- String idValue = (String)iter.next();
- if (j > 0)
- sb.append(',');
- sb.append('?');
- list.add(idValue);
+ list.add(iter.next());
j++;
}
if (j > 0)
- findRowIdsForDocIds(outputConnectionName,rowIDSet,list,sb.toString());
+ findRowIdsForDocIds(outputConnectionName,rowIDSet,list);
// Now, break row id's into chunks too; submit one chunk at a time
j = 0;
list.clear();
- sb.setLength(0);
iter = rowIDSet.keySet().iterator();
+ maxClauses = maxClausesUpdateRowIds();
while (iter.hasNext())
{
- if (j == maxInClause)
+ if (j == maxClauses)
{
- updateRowIds(list,sb.toString(),checkTime);
+ updateRowIds(list,checkTime);
list.clear();
- sb.setLength(0);
j = 0;
}
- Long idValue = (Long)iter.next();
- if (j > 0)
- sb.append(',');
- sb.append('?');
- list.add(idValue);
+ list.add(iter.next());
j++;
}
if (j > 0)
- updateRowIds(list,sb.toString(),checkTime);
+ updateRowIds(list,checkTime);
}
catch (ManifoldCFException e)
{
@@ -618,14 +616,25 @@ public class IncrementalIngester extends
documentCheckMultiple(outputConnectionName,new String[]{identifierClass},new String[]{identifierHash},checkTime);
}
+ /** Calculate the number of clauses.
+ */
+ protected int maxClausesUpdateRowIds()
+ {
+ return findConjunctionClauseMax(new ClauseDescription[]{});
+ }
+
/** Update a chunk of row ids.
*/
- protected void updateRowIds(ArrayList list, String queryPart, long checkTime)
+ protected void updateRowIds(ArrayList list, long checkTime)
throws ManifoldCFException
{
+ ArrayList newList = new ArrayList();
+ String query = buildConjunctionClause(newList,new ClauseDescription[]{
+ new MultiClause(idField,list)});
+
HashMap map = new HashMap();
map.put(lastIngestField,new Long(checkTime));
- performUpdate(map,"WHERE "+idField+" IN ("+queryPart+")",list,null);
+ performUpdate(map,"WHERE "+query,newList,null);
}
/** Delete multiple documents from the search engine index.
@@ -755,8 +764,8 @@ public class IncrementalIngester extends
// so the first step is to come up with ALL the doc hash values before looping
// over them.
- int maxInClause = getMaxInClause();
-
+ int maxClauses;
+
// Find all the documents that match this set of URIs
HashMap docURIHashValues = new HashMap();
HashMap docURIValues = new HashMap();
@@ -775,51 +784,41 @@ public class IncrementalIngester extends
Iterator iter = docURIHashValues.keySet().iterator();
j = 0;
ArrayList hashList = new ArrayList();
- StringBuilder sb = new StringBuilder();
+ maxClauses = maxClausesRowIdsForURIs(outputConnectionName);
while (iter.hasNext())
{
- if (j == maxInClause)
+ if (j == maxClauses)
{
- findRowIdsForURIs(outputConnectionName,rowIDSet,docURIValues,hashList,sb.toString());
+ findRowIdsForURIs(outputConnectionName,rowIDSet,docURIValues,hashList);
hashList.clear();
- sb.setLength(0);
j = 0;
}
- String hashValue = (String)iter.next();
- if (j > 0)
- sb.append(',');
- sb.append('?');
- hashList.add(hashValue);
+ hashList.add(iter.next());
j++;
}
if (j > 0)
- findRowIdsForURIs(outputConnectionName,rowIDSet,docURIValues,hashList,sb.toString());
+ findRowIdsForURIs(outputConnectionName,rowIDSet,docURIValues,hashList);
// Next, go through the list of row IDs, and delete them in chunks
j = 0;
ArrayList list = new ArrayList();
- sb.setLength(0);
iter = rowIDSet.keySet().iterator();
+ maxClauses = maxClausesDeleteRowIds();
while (iter.hasNext())
{
- if (j == maxInClause)
+ if (j == maxClauses)
{
- deleteRowIds(list,sb.toString());
+ deleteRowIds(list);
list.clear();
- sb.setLength(0);
j = 0;
}
- Long idValue = (Long)iter.next();
- if (j > 0)
- sb.append(',');
- sb.append('?');
- list.add(idValue);
+ list.add(iter.next());
j++;
}
if (j > 0)
- deleteRowIds(list,sb.toString());
+ deleteRowIds(list);
// Now, find the set of documents that remain that match the document identifiers.
HashMap docIdValues = new HashMap();
@@ -837,51 +836,41 @@ public class IncrementalIngester extends
iter = docIdValues.keySet().iterator();
j = 0;
list.clear();
- sb.setLength(0);
+ maxClauses = maxClausesRowIdsForDocIds(outputConnectionName);
while (iter.hasNext())
{
- if (j == maxInClause)
+ if (j == maxClauses)
{
- findRowIdsForDocIds(outputConnectionName,rowIDSet,list,sb.toString());
+ findRowIdsForDocIds(outputConnectionName,rowIDSet,list);
list.clear();
- sb.setLength(0);
j = 0;
}
- String hashValue = (String)iter.next();
- if (j > 0)
- sb.append(',');
- sb.append('?');
- list.add(hashValue);
+ list.add(iter.next());
j++;
}
if (j > 0)
- findRowIdsForDocIds(outputConnectionName,rowIDSet,list,sb.toString());
+ findRowIdsForDocIds(outputConnectionName,rowIDSet,list);
// Next, go through the list of row IDs, and delete them in chunks
j = 0;
list.clear();
- sb.setLength(0);
iter = rowIDSet.keySet().iterator();
+ maxClauses = maxClausesDeleteRowIds();
while (iter.hasNext())
{
- if (j == maxInClause)
+ if (j == maxClauses)
{
- deleteRowIds(list,sb.toString());
+ deleteRowIds(list);
list.clear();
- sb.setLength(0);
j = 0;
}
- Long idValue = (Long)iter.next();
- if (j > 0)
- sb.append(',');
- sb.append('?');
- list.add(idValue);
+ list.add(iter.next());
j++;
}
if (j > 0)
- deleteRowIds(list,sb.toString());
+ deleteRowIds(list);
}
catch (ManifoldCFException e)
@@ -905,16 +894,28 @@ public class IncrementalIngester extends
}
}
+ /** Calculate the clauses.
+ */
+ protected int maxClausesRowIdsForURIs(String outputConnectionName)
+ {
+ return findConjunctionClauseMax(new ClauseDescription[]{
+ new UnitaryClause(outputConnNameField,outputConnectionName)});
+ }
+
/** Given values and parameters corresponding to a set of hash values, add corresponding
* table row id's to the output map.
*/
- protected void findRowIdsForURIs(String outputConnectionName, HashMap rowIDSet, HashMap uris, ArrayList hashParamValues,
- String paramList)
+ protected void findRowIdsForURIs(String outputConnectionName, HashMap rowIDSet, HashMap uris, ArrayList hashParamValues)
throws ManifoldCFException
{
- hashParamValues.add(outputConnectionName);
+ ArrayList list = new ArrayList();
+ String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new MultiClause(uriHashField,hashParamValues),
+ new UnitaryClause(outputConnNameField,outputConnectionName)});
+
IResultSet set = performQuery("SELECT "+idField+","+docURIField+" FROM "+
- getTableName()+" WHERE "+uriHashField+" IN ("+paramList+") AND "+outputConnNameField+"=?",hashParamValues,null,null);
+ getTableName()+" WHERE "+query,list,null,null);
+
int i = 0;
while (i < set.getRowCount())
{
@@ -931,16 +932,28 @@ public class IncrementalIngester extends
}
}
+ /** Calculate the maximum number of doc ids we should use.
+ */
+ protected int maxClausesRowIdsForDocIds(String outputConnectionName)
+ {
+ return findConjunctionClauseMax(new ClauseDescription[]{
+ new UnitaryClause(outputConnNameField,outputConnectionName)});
+ }
+
/** Given values and parameters corresponding to a set of hash values, add corresponding
* table row id's to the output map.
*/
- protected void findRowIdsForDocIds(String outputConnectionName, HashMap rowIDSet, ArrayList paramValues,
- String paramList)
+ protected void findRowIdsForDocIds(String outputConnectionName, HashMap rowIDSet, ArrayList paramValues)
throws ManifoldCFException
{
- paramValues.add(outputConnectionName);
+ ArrayList list = new ArrayList();
+ String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new MultiClause(docKeyField,paramValues),
+ new UnitaryClause(outputConnNameField,outputConnectionName)});
+
IResultSet set = performQuery("SELECT "+idField+" FROM "+
- getTableName()+" WHERE "+docKeyField+" IN ("+paramList+") AND "+outputConnNameField+"=?",paramValues,null,null);
+ getTableName()+" WHERE "+query,list,null,null);
+
int i = 0;
while (i < set.getRowCount())
{
@@ -950,12 +963,22 @@ public class IncrementalIngester extends
}
}
+ /** Calculate the maximum number of clauses.
+ */
+ protected int maxClausesDeleteRowIds()
+ {
+ return findConjunctionClauseMax(new ClauseDescription[]{});
+ }
+
/** Delete a chunk of row ids.
*/
- protected void deleteRowIds(ArrayList list, String queryPart)
+ protected void deleteRowIds(ArrayList list)
throws ManifoldCFException
{
- performDelete("WHERE "+idField+" IN ("+queryPart+")",list,null);
+ ArrayList newList = new ArrayList();
+ String query = buildConjunctionClause(newList,new ClauseDescription[]{
+ new MultiClause(idField,list)});
+ performDelete("WHERE "+query,newList,null);
}
/** Delete a document from the search engine index.
@@ -993,28 +1016,23 @@ public class IncrementalIngester extends
beginTransaction();
try
{
- StringBuilder sb = new StringBuilder();
ArrayList list = new ArrayList();
- int maxCount = getMaxInClause();
+ int maxCount = maxClauseDocumentURIChunk(outputConnectionName);
int j = 0;
Iterator iter = map.keySet().iterator();
while (iter.hasNext())
{
if (j == maxCount)
{
- getDocumentURIChunk(rval,map,outputConnectionName,sb.toString(),list);
+ getDocumentURIChunk(rval,map,outputConnectionName,list);
j = 0;
- sb.setLength(0);
list.clear();
}
- if (j > 0)
- sb.append(',');
- sb.append('?');
list.add(iter.next());
j++;
}
if (j > 0)
- getDocumentURIChunk(rval,map,outputConnectionName,sb.toString(),list);
+ getDocumentURIChunk(rval,map,outputConnectionName,list);
return rval;
}
catch (ManifoldCFException e)
@@ -1116,28 +1134,23 @@ public class IncrementalIngester extends
beginTransaction();
try
{
- StringBuilder sb = new StringBuilder();
ArrayList list = new ArrayList();
- int maxCount = getMaxInClause();
+ int maxCount = maxClauseDocumentIngestDataChunk(outputConnectionName);
int j = 0;
Iterator iter = indexMap.keySet().iterator();
while (iter.hasNext())
{
if (j == maxCount)
{
- getDocumentIngestDataChunk(rval,indexMap,outputConnectionName,sb.toString(),list);
+ getDocumentIngestDataChunk(rval,indexMap,outputConnectionName,list);
j = 0;
- sb.setLength(0);
list.clear();
}
- if (j > 0)
- sb.append(',');
- sb.append('?');
list.add(iter.next());
j++;
}
if (j > 0)
- getDocumentIngestDataChunk(rval,indexMap,outputConnectionName,sb.toString(),list);
+ getDocumentIngestDataChunk(rval,indexMap,outputConnectionName,list);
return rval;
}
catch (ManifoldCFException e)
@@ -1212,51 +1225,56 @@ public class IncrementalIngester extends
}
// Get the chunk size
- int maxInClause = getMaxInClause();
-
+ int maxClause = maxClauseGetIntervals(outputConnectionName);
// Loop through the hash codes
Iterator iter = idCodes.keySet().iterator();
ArrayList list = new ArrayList();
- StringBuilder sb = new StringBuilder();
j = 0;
while (iter.hasNext())
{
- if (j == maxInClause)
+ if (j == maxClause)
{
- getIntervals(rval,outputConnectionName,list,sb.toString(),returnMap);
+ getIntervals(rval,outputConnectionName,list,returnMap);
list.clear();
- sb.setLength(0);
j = 0;
}
- if (j > 0)
- sb.append(',');
- sb.append('?');
list.add(iter.next());
j++;
}
if (j > 0)
- getIntervals(rval,outputConnectionName,list,sb.toString(),returnMap);
+ getIntervals(rval,outputConnectionName,list,returnMap);
return rval;
}
-
+ /** Calculate the number of clauses.
+ */
+ protected int maxClauseGetIntervals(String outputConnectionName)
+ {
+ return findConjunctionClauseMax(new ClauseDescription[]{
+ new UnitaryClause(outputConnNameField,outputConnectionName)});
+ }
+
/** Query for and calculate the interval for a bunch of hashcodes.
*@param rval is the array to stuff calculated return values into.
*@param list is the list of parameters.
*@param queryPart is the part of the query pertaining to the list of hashcodes
*@param returnMap is a mapping from document id to rval index.
*/
- protected void getIntervals(long[] rval, String outputConnectionName, ArrayList list, String queryPart, HashMap returnMap)
+ protected void getIntervals(long[] rval, String outputConnectionName, ArrayList list, HashMap returnMap)
throws ManifoldCFException
{
- list.add(outputConnectionName);
+ ArrayList newList = new ArrayList();
+ String query = buildConjunctionClause(newList,new ClauseDescription[]{
+ new MultiClause(docKeyField,list),
+ new UnitaryClause(outputConnNameField,outputConnectionName)});
+
IResultSet set = performQuery("SELECT "+docKeyField+","+changeCountField+","+firstIngestField+","+lastIngestField+
- " FROM "+getTableName()+" WHERE "+
- docKeyField+" IN("+queryPart+") AND "+outputConnNameField+"=?",list,null,null);
+ " FROM "+getTableName()+" WHERE "+query,newList,null,null);
+
int i = 0;
while (i < set.getRowCount())
{
@@ -1286,8 +1304,10 @@ public class IncrementalIngester extends
HashMap map = new HashMap();
map.put(lastVersionField,null);
ArrayList list = new ArrayList();
- list.add(outputConnectionName);
- performUpdate(map,"WHERE "+outputConnNameField+"=?",list,null);
+ String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(outputConnNameField,outputConnectionName)});
+
+ performUpdate(map,"WHERE "+query,list,null);
}
/** Note the ingestion of a document, or the "update" of a document.
@@ -1353,10 +1373,11 @@ public class IncrementalIngester extends
{
// Look for existing row.
ArrayList list = new ArrayList();
- list.add(docKey);
- list.add(outputConnectionName);
+ String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(docKeyField,docKey),
+ new UnitaryClause(outputConnNameField,outputConnectionName)});
IResultSet set = performQuery("SELECT "+idField+","+changeCountField+" FROM "+getTableName()+" WHERE "+
- docKeyField+"=? AND "+outputConnNameField+"=? FOR UPDATE",list,null,null);
+ query+" FOR UPDATE",list,null,null);
IResultRow row = null;
if (set.getRowCount() > 0)
row = set.getRow(0);
@@ -1365,11 +1386,12 @@ public class IncrementalIngester extends
{
// Update the record
list.clear();
- list.add(row.getValue(idField));
+ query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(idField,row.getValue(idField))});
long changeCount = ((Long)row.getValue(changeCountField)).longValue();
changeCount++;
map.put(changeCountField,new Long(changeCount));
- performUpdate(map,"WHERE "+idField+"=?",list,null);
+ performUpdate(map,"WHERE "+query,list,null);
// Update successful!
return;
}
@@ -1452,18 +1474,31 @@ public class IncrementalIngester extends
}
}
+ /** Calculate how many clauses at a time
+ */
+ protected int maxClauseDocumentURIChunk(String outputConnectionName)
+ {
+ return findConjunctionClauseMax(new ClauseDescription[]{
+ new UnitaryClause(outputConnNameField,outputConnectionName)});
+ }
+
/** Get a chunk of document uris.
*@param rval is the string array where the uris should be put.
*@param map is the map from id to index.
*@param clause is the in clause for the query.
*@param list is the parameter list for the query.
*/
- protected void getDocumentURIChunk(DeleteInfo[] rval, Map map, String outputConnectionName, String clause, ArrayList list)
+ protected void getDocumentURIChunk(DeleteInfo[] rval, Map map, String outputConnectionName, ArrayList list)
throws ManifoldCFException
{
- list.add(outputConnectionName);
+ ArrayList newList = new ArrayList();
+ String query = buildConjunctionClause(newList,new ClauseDescription[]{
+ new MultiClause(docKeyField,list),
+ new UnitaryClause(outputConnNameField,outputConnectionName)});
+
IResultSet set = performQuery("SELECT "+docKeyField+","+docURIField+","+lastOutputVersionField+" FROM "+getTableName()+" WHERE "+
- docKeyField+" IN ("+clause+") AND "+outputConnNameField+"=?",list,null,null);
+ query,newList,null,null);
+
// Go through list and put into buckets.
int i = 0;
while (i < set.getRowCount())
@@ -1482,19 +1517,31 @@ public class IncrementalIngester extends
}
}
+ /** Count the clauses
+ */
+ protected int maxClauseDocumentIngestDataChunk(String outputConnectionName)
+ {
+ return findConjunctionClauseMax(new ClauseDescription[]{
+ new UnitaryClause(outputConnNameField,outputConnectionName)});
+ }
+
/** Get a chunk of document ingest data records.
*@param rval is the document ingest status array where the data should be put.
*@param map is the map from id to index.
*@param clause is the in clause for the query.
*@param list is the parameter list for the query.
*/
- protected void getDocumentIngestDataChunk(DocumentIngestStatus[] rval, Map map, String outputConnectionName, String clause, ArrayList list)
+ protected void getDocumentIngestDataChunk(DocumentIngestStatus[] rval, Map map, String outputConnectionName, ArrayList list)
throws ManifoldCFException
{
+ ArrayList newList = new ArrayList();
+ String query = buildConjunctionClause(newList,new ClauseDescription[]{
+ new MultiClause(docKeyField,list),
+ new UnitaryClause(outputConnNameField,outputConnectionName)});
+
// Get the primary records associated with this hash value
- list.add(outputConnectionName);
IResultSet set = performQuery("SELECT "+idField+","+docKeyField+","+lastVersionField+","+lastOutputVersionField+","+authorityNameField+" FROM "+getTableName()+" WHERE "+
- docKeyField+" IN ("+clause+") AND "+outputConnNameField+"=?",list,null,null);
+ query,newList,null,null);
// Now, go through the original request once more, this time building the result
int i = 0;
Modified: incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java (original)
+++ incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java Mon Oct 31 14:26:33 2011
@@ -102,7 +102,27 @@ public class OutputConnectionManager ext
// Upgrade code, if needed, goes here.
}
- // Index management code goes here.
+ // Index management
+ IndexDescription classIndex = new IndexDescription(false,new String[]{classNameField});
+
+ // Get rid of indexes that shouldn't be there
+ Map indexes = getTableIndexes(null,null);
+ Iterator iter = indexes.keySet().iterator();
+ while (iter.hasNext())
+ {
+ String indexName = (String)iter.next();
+ IndexDescription id = (IndexDescription)indexes.get(indexName);
+
+ if (classIndex != null && id.equals(classIndex))
+ classIndex = null;
+ else if (indexName.indexOf("_pkey") == -1)
+ // This index shouldn't be here; drop it
+ performRemoveIndex(indexName);
+ }
+
+ // Add the ones we didn't find
+ if (classIndex != null)
+ performAddIndex(null,classIndex);
break;
}
@@ -261,6 +281,7 @@ public class OutputConnectionManager ext
while (true)
{
// Catch deadlock condition
+ long sleepAmt = 0L;
try
{
ICacheHandle ch = cacheManager.enterCache(null,cacheKeys,getTransactionID());
@@ -275,9 +296,10 @@ public class OutputConnectionManager ext
boolean isNew = object.getIsNew();
// See whether the instance exists
ArrayList params = new ArrayList();
- params.add(object.getName());
+ String query = buildConjunctionClause(params,new ClauseDescription[]{
+ new UnitaryClause(nameField,object.getName())});
IResultSet set = performQuery("SELECT * FROM "+getTableName()+" WHERE "+
- nameField+"=? FOR UPDATE",params,null,null);
+ query+" FOR UPDATE",params,null,null);
HashMap values = new HashMap();
values.put(descriptionField,object.getDescription());
values.put(classNameField,object.getClassName());
@@ -300,8 +322,9 @@ public class OutputConnectionManager ext
// Update
params.clear();
- params.add(object.getName());
- performUpdate(values," WHERE "+nameField+"=?",params,null);
+ query = buildConjunctionClause(params,new ClauseDescription[]{
+ new UnitaryClause(nameField,object.getName())});
+ performUpdate(values," WHERE "+query,params,null);
}
else
{
@@ -347,14 +370,11 @@ public class OutputConnectionManager ext
// Is this a deadlock exception? If so, we want to try again.
if (e.getErrorCode() != ManifoldCFException.DATABASE_TRANSACTION_ABORT)
throw e;
- try
- {
- ManifoldCF.sleep((long)(random.nextDouble() * 60000.0 + 500.0));
- }
- catch (InterruptedException e2)
- {
- throw new ManifoldCFException(e2.getMessage(),e2,ManifoldCFException.INTERRUPTED);
- }
+ sleepAmt = getSleepAmt();
+ }
+ finally
+ {
+ sleepFor(sleepAmt);
}
}
}
@@ -381,8 +401,9 @@ public class OutputConnectionManager ext
throw new ManifoldCFException("Can't delete output connection '"+name+"': existing entities refer to it");
ManifoldCF.noteConfigurationChange();
ArrayList params = new ArrayList();
- params.add(name);
- performDelete("WHERE "+nameField+"=?",params,null);
+ String query = buildConjunctionClause(params,new ClauseDescription[]{
+ new UnitaryClause(nameField,name)});
+ performDelete("WHERE "+query,params,null);
cacheManager.invalidateKeys(ch);
}
catch (ManifoldCFException e)
@@ -418,8 +439,9 @@ public class OutputConnectionManager ext
ssb.add(getOutputConnectionsKey());
StringSet localCacheKeys = new StringSet(ssb);
ArrayList params = new ArrayList();
- params.add(className);
- IResultSet set = performQuery("SELECT "+nameField+" FROM "+getTableName()+" WHERE "+classNameField+"=?",params,
+ String query = buildConjunctionClause(params,new ClauseDescription[]{
+ new UnitaryClause(classNameField,className)});
+ IResultSet set = performQuery("SELECT "+nameField+" FROM "+getTableName()+" WHERE "+query,params,
localCacheKeys,null);
String[] rval = new String[set.getRowCount()];
int i = 0;
@@ -447,8 +469,9 @@ public class OutputConnectionManager ext
ssb.add(getOutputConnectionKey(name));
StringSet localCacheKeys = new StringSet(ssb);
ArrayList params = new ArrayList();
- params.add(name);
- IResultSet set = performQuery("SELECT "+classNameField+" FROM "+getTableName()+" WHERE "+nameField+"=?",params,
+ String query = buildConjunctionClause(params,new ClauseDescription[]{
+ new UnitaryClause(nameField,name)});
+ IResultSet set = performQuery("SELECT "+classNameField+" FROM "+getTableName()+" WHERE "+query,params,
localCacheKeys,null);
if (set.getRowCount() == 0)
throw new ManifoldCFException("No such connection: '"+name+"'");
@@ -524,28 +547,23 @@ public class OutputConnectionManager ext
try
{
i = 0;
- StringBuilder sb = new StringBuilder();
ArrayList params = new ArrayList();
int j = 0;
- int maxIn = getMaxInClause();
+ int maxIn = maxClauseGetOutputConnectionsChunk();
while (i < connectionNames.length)
{
if (j == maxIn)
{
- getOutputConnectionsChunk(rval,returnIndex,sb.toString(),params);
- sb.setLength(0);
+ getOutputConnectionsChunk(rval,returnIndex,params);
params.clear();
j = 0;
}
- if (j > 0)
- sb.append(',');
- sb.append('?');
params.add(connectionNames[i]);
i++;
j++;
}
if (j > 0)
- getOutputConnectionsChunk(rval,returnIndex,sb.toString(),params);
+ getOutputConnectionsChunk(rval,returnIndex,params);
return rval;
}
catch (Error e)
@@ -564,18 +582,26 @@ public class OutputConnectionManager ext
}
}
+ /** Calculate max number of clauses to send to getOutputConnectionsChunk.
+ */
+ protected int maxClauseGetOutputConnectionsChunk()
+ {
+ return findConjunctionClauseMax(new ClauseDescription[]{});
+ }
+
/** Read a chunk of output connections.
*@param rval is the place to put the read policies.
*@param returnIndex is a map from the object id (resource id) and the rval index.
- *@param idList is the list of id's.
*@param params is the set of parameters.
*/
- protected void getOutputConnectionsChunk(OutputConnection[] rval, Map returnIndex, String idList, ArrayList params)
+ protected void getOutputConnectionsChunk(OutputConnection[] rval, Map returnIndex, ArrayList params)
throws ManifoldCFException
{
- IResultSet set;
- set = performQuery("SELECT * FROM "+getTableName()+" WHERE "+
- nameField+" IN ("+idList+")",params,null,null);
+ ArrayList list = new ArrayList();
+ String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new MultiClause(nameField,params)});
+ IResultSet set = performQuery("SELECT * FROM "+getTableName()+" WHERE "+
+ query,list,null,null);
int i = 0;
while (i < set.getRowCount())
{
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java Mon Oct 31 14:26:33 2011
@@ -364,6 +364,25 @@ public class BaseTable
orderFields,orderFieldsAscending,otherFields);
}
+ /* Calculate the number of values a particular clause can have, given the values for all the other clauses.
+ * For example, if in the expression x AND y AND z, x has 2 values and z has 1, find out how many values x can legally have
+ * when using the buildConjunctionClause() method below.
+ */
+ public int findConjunctionClauseMax(ClauseDescription[] otherClauseDescriptions)
+ {
+ return dbInterface.findConjunctionClauseMax(otherClauseDescriptions);
+ }
+
+ /* Construct a conjunction clause, e.g. x AND y AND z, where there is expected to be an index (x,y,z,...), and where x, y, or z
+ * can have multiple distinct values, The proper implementation of this method differs from database to database, because some databases
+ * only permit index operations when there are OR's between clauses, such as x1 AND y1 AND z1 OR x2 AND y2 AND z2 ..., where others
+ * only recognize index operations when there are lists specified for each, such as x IN (x1,x2) AND y IN (y1,y2) AND z IN (z1,z2).
+ */
+ public String buildConjunctionClause(List outputParameters, ClauseDescription[] clauseDescriptions)
+ {
+ return dbInterface.buildConjunctionClause(outputParameters,clauseDescriptions);
+ }
+
/** Obtain the maximum number of individual items that should be
* present in an IN clause. Exceeding this amount will potentially cause the query performance
* to drop.
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java Mon Oct 31 14:26:33 2011
@@ -108,6 +108,16 @@ public class DBInterfaceDerby extends Da
this.password = password;
}
+ public DBInterfaceDerby(IThreadContext tc, String databaseName)
+ throws ManifoldCFException
+ {
+ super(tc,_url+databaseName,_driver,databaseName,"","");
+ cacheKey = CacheKeyFactory.makeDatabaseKey(this.databaseName);
+ lockManager = LockManagerFactory.make(tc);
+ this.userName = "";
+ this.password = "";
+ }
+
/** Initialize. This method is called once per JVM instance, in order to set up
* database communication.
*/
@@ -704,7 +714,7 @@ public class DBInterfaceDerby extends Da
public void createUserAndDatabase(String adminUserName, String adminPassword, StringSet invalidateKeys)
throws ManifoldCFException
{
- Database rootDatabase = new Database(context,_url+databaseName,_driver,databaseName,"","");
+ Database rootDatabase = new DBInterfaceDerby(context,databaseName);
IResultSet set = rootDatabase.executeQuery("VALUES SYSCS_UTIL.SYSCS_GET_DATABASE_PROPERTY('derby.user."+userName+"')",null,null,null,null,true,-1,null,null);
if (set.getRowCount() == 0)
{
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java Mon Oct 31 14:26:33 2011
@@ -1000,6 +1000,104 @@ public class DBInterfaceHSQLDB extends D
return sb.toString();
}
+ /* Calculate the number of values a particular clause can have, given the values for all the other clauses.
+ * For example, if in the expression x AND y AND z, x has 2 values and z has 1, find out how many values x can legally have
+ * when using the buildConjunctionClause() method below.
+ */
+ @Override
+ public int findConjunctionClauseMax(ClauseDescription[] otherClauseDescriptions)
+ {
+ // Special handling when there's only 1
+ if (otherClauseDescriptions.length == 0)
+ return super.findConjunctionClauseMax(otherClauseDescriptions);
+
+ // Since it's an OR clause we have to generate, figure out how many clauses are generated by the others,
+ // and work back from there.
+ int number = 1;
+ for (int i = 0 ; i < otherClauseDescriptions.length ; i++)
+ {
+ ClauseDescription otherClause = otherClauseDescriptions[i];
+ List values = otherClause.getValues();
+ if (values != null)
+ number *= values.size();
+ }
+ int rval = getMaxOrClause() / number;
+ if (rval == 0)
+ rval = 1;
+ return rval;
+ }
+
+ /* Construct a conjunction clause, e.g. x AND y AND z, where there is expected to be an index (x,y,z,...), and where x, y, or z
+ * can have multiple distinct values, The proper implementation of this method differs from database to database, because some databases
+ * only permit index operations when there are OR's between clauses, such as x1 AND y1 AND z1 OR x2 AND y2 AND z2 ..., where others
+ * only recognize index operations when there are lists specified for each, such as x IN (x1,x2) AND y IN (y1,y2) AND z IN (z1,z2).
+ */
+ @Override
+ public String buildConjunctionClause(List outputParameters, ClauseDescription[] clauseDescriptions)
+ {
+ // Special handling when there's only 1
+ if (clauseDescriptions.length == 1)
+ return super.buildConjunctionClause(outputParameters,clauseDescriptions);
+
+ StringBuilder sb = new StringBuilder("(");
+ int[] counters = new int[clauseDescriptions.length];
+ for (int i = 0 ; i < counters.length ; i++)
+ {
+ counters[i] = 0;
+ }
+
+ boolean isFirst = true;
+ while (true)
+ {
+ // Add this clause in
+ if (isFirst)
+ isFirst = false;
+ else
+ sb.append(" OR ");
+ for (int i = 0 ; i < counters.length ; i++)
+ {
+ ClauseDescription cd = clauseDescriptions[i];
+ if (i > 0)
+ sb.append(" AND ");
+ List values = cd.getValues();
+ String joinColumn = cd.getJoinColumnName();
+ sb.append(cd.getColumnName()).append(cd.getOperation());
+ if (values != null)
+ {
+ sb.append("?");
+ outputParameters.add(values.get(counters[i]));
+ }
+ else if (joinColumn != null)
+ sb.append(joinColumn);
+ }
+
+ // Now, increment the counters
+ int j = 0;
+ while (true)
+ {
+ if (j == counters.length)
+ {
+ sb.append(")");
+ return sb.toString();
+ }
+ counters[j]++;
+ ClauseDescription cd = clauseDescriptions[j];
+ List values = cd.getValues();
+ int size = 1;
+ if (values != null)
+ size = values.size();
+ if (counters[j] < size)
+ break;
+ j++;
+ for (int k = 0 ; k < j ; k++)
+ {
+ counters[k] = 0;
+ }
+ // Loop around to carry a one to the j'th counter
+ }
+ }
+ }
+
/** Obtain the maximum number of individual items that should be
* present in an IN clause. Exceeding this amount will potentially cause the query performance
* to drop.
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java Mon Oct 31 14:26:33 2011
@@ -387,7 +387,7 @@ public class DBInterfaceMySQL extends Da
throws ManifoldCFException
{
// Connect to super database
- Database masterDatabase = new Database(context,_url+"mysql",_driver,"mysql",adminUserName,adminPassword);
+ Database masterDatabase = new DBInterfaceMySQL(context,"mysql",adminUserName,adminPassword);
List list = new ArrayList();
list.add("utf8");
masterDatabase.executeQuery("CREATE DATABASE "+databaseName+" CHARACTER SET ?",list,
@@ -412,7 +412,7 @@ public class DBInterfaceMySQL extends Da
throws ManifoldCFException
{
// Connect to super database
- Database masterDatabase = new Database(context,_url+"mysql",_driver,"mysql",adminUserName,adminPassword);
+ Database masterDatabase = new DBInterfaceMySQL(context,"mysql",adminUserName,adminPassword);
masterDatabase.executeQuery("DROP DATABASE "+databaseName,null,null,invalidateKeys,null,false,0,null,null);
}
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java Mon Oct 31 14:26:33 2011
@@ -523,7 +523,7 @@ public class DBInterfacePostgreSQL exten
throws ManifoldCFException
{
// Create a connection to the master database, using the credentials supplied
- Database masterDatabase = new Database(context,getJdbcUrl("template1"),_driver,"template1",adminUserName,adminPassword);
+ Database masterDatabase = new DBInterfacePostgreSQL(context,"template1",adminUserName,adminPassword);
try
{
// Create user
@@ -577,7 +577,7 @@ public class DBInterfacePostgreSQL exten
throws ManifoldCFException
{
// Create a connection to the master database, using the credentials supplied
- Database masterDatabase = new Database(context,getJdbcUrl("template1"),_driver,"template1",adminUserName,adminPassword);
+ Database masterDatabase = new DBInterfacePostgreSQL(context,"template1",adminUserName,adminPassword);
try
{
// Drop database
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java Mon Oct 31 14:26:33 2011
@@ -33,7 +33,7 @@ import javax.sql.*;
* the transaction management will get screwed up (i.e. nobody will know what happened to the connection
* handles...)
*/
-public class Database
+public abstract class Database
{
public static final String _rcsid = "@(#)$Id: Database.java 988245 2010-08-23 18:39:35Z kwright $";
@@ -50,6 +50,8 @@ public class Database
protected int delayedTransactionDepth = 0;
protected Map<String,Modifications> modificationsSet = new HashMap<String,Modifications>();
+ protected long maxQueryTime;
+
protected static Random random = new Random();
protected final static String _TRANSACTION_ = "_TRANSACTION_";
@@ -63,6 +65,8 @@ public class Database
this.databaseName = databaseName;
this.userName = userName;
this.password = password;
+
+ this.maxQueryTime = ((long)ManifoldCF.getIntProperty(ManifoldCF.databaseQueryMaxTimeProperty,60)) * 1000L;
this.cacheManager = CacheManagerFactory.make(context);
}
@@ -426,6 +430,71 @@ public class Database
}
}
+ /* Calculate the number of values a particular clause can have, given the values for all the other clauses.
+ * For example, if in the expression x AND y AND z, x has 2 values and z has 1, find out how many values x can legally have
+ * when using the buildConjunctionClause() method below.
+ */
+ public int findConjunctionClauseMax(ClauseDescription[] otherClauseDescriptions)
+ {
+ // Base implementation uses "IN" for multiple values, since this seems to be widely accepted.
+ return getMaxInClause();
+ }
+
+ /** Obtain the maximum number of individual items that should be
+ * present in an IN clause. Exceeding this amount will potentially cause the query performance
+ * to drop.
+ *@return the maximum number of IN clause members.
+ */
+ public abstract int getMaxInClause();
+
+ /* Construct a conjunction clause, e.g. x AND y AND z, where there is expected to be an index (x,y,z,...), and where x, y, or z
+ * can have multiple distinct values, The proper implementation of this method differs from database to database, because some databases
+ * only permit index operations when there are OR's between clauses, such as x1 AND y1 AND z1 OR x2 AND y2 AND z2 ..., where others
+ * only recognize index operations when there are lists specified for each, such as x IN (x1,x2) AND y IN (y1,y2) AND z IN (z1,z2).
+ */
+ public String buildConjunctionClause(List outputParameters, ClauseDescription[] clauseDescriptions)
+ {
+ // Base implementation uses "IN" for multiple values, since this seems to be widely accepted.
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0 ; i < clauseDescriptions.length ; i++)
+ {
+ ClauseDescription cd = clauseDescriptions[i];
+ if (i > 0)
+ sb.append(" AND ");
+ sb.append(cd.getColumnName());
+ String operation = cd.getOperation();
+ List values = cd.getValues();
+ String joinColumn = cd.getJoinColumnName();
+ if (values != null)
+ {
+ if (values.size() > 1)
+ {
+ sb.append(" IN (");
+ for (int j = 0 ; j < values.size() ; j++)
+ {
+ if (j > 0)
+ sb.append(",");
+ sb.append("?");
+ outputParameters.add(values.get(j));
+ }
+ sb.append(")");
+ }
+ else
+ {
+ sb.append(operation).append("?");
+ outputParameters.add(values.get(0));
+ }
+ }
+ else if (joinColumn != null)
+ {
+ sb.append(operation).append(joinColumn);
+ }
+ else
+ sb.append(operation);
+ }
+ return sb.toString();
+ }
+
/** Class to keep track of modifications while we're in a transaction.
*/
protected static class Modifications
@@ -1231,10 +1300,10 @@ public class Database
description.getMaxReturn(),spec,limit);
long endTime = System.currentTimeMillis();
- if (endTime-startTime > 60000L && description.getQuery().length() >= 6 &&
+ if (endTime-startTime > database.maxQueryTime && description.getQuery().length() >= 6 &&
("SELECT".equalsIgnoreCase(description.getQuery().substring(0,6)) || "UPDATE".equalsIgnoreCase(description.getQuery().substring(0,6))))
{
- Logging.db.warn("Found a query that took more than a minute ("+new Long(endTime-startTime).toString()+" ms): ["+description.getQuery()+"]");
+ Logging.db.warn("Found a long-running query ("+new Long(endTime-startTime).toString()+" ms): ["+description.getQuery()+"]");
if (description.getParameters() != null)
{
int j = 0;
Added: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ClauseDescription.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ClauseDescription.java?rev=1195471&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ClauseDescription.java (added)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ClauseDescription.java Mon Oct 31 14:26:33 2011
@@ -0,0 +1,39 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.core.interfaces;
+
+import java.util.*;
+
+public interface ClauseDescription
+{
+ public static final String _rcsid = "@(#)$Id$";
+
+ /** Get the column name */
+ public String getColumnName();
+
+ /** Get the operation string */
+ public String getOperation();
+
+ /** Get the individual values */
+ public List getValues();
+
+ /** Get the join column name */
+ public String getJoinColumnName();
+
+}
Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ClauseDescription.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ClauseDescription.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java Mon Oct 31 14:26:33 2011
@@ -302,6 +302,19 @@ public interface IDBInterface
public String constructDistinctOnClause(List outputParameters, String baseQuery, List baseParameters,
String[] distinctFields, String[] orderFields, boolean[] orderFieldsAscending, Map<String,String> otherFields);
+ /* Calculate the number of values a particular clause can have, given the values for all the other clauses.
+ * For example, if in the expression x AND y AND z, x has 2 values and z has 1, find out how many values x can legally have
+ * when using the buildConjunctionClause() method below.
+ */
+ public int findConjunctionClauseMax(ClauseDescription[] otherClauseDescriptions);
+
+ /* Construct a conjunction clause, e.g. x AND y AND z, where there is expected to be an index (x,y,z,...), and where x, y, or z
+ * can have multiple distinct values, The proper implementation of this method differs from database to database, because some databases
+ * only permit index operations when there are OR's between clauses, such as x1 AND y1 AND z1 OR x2 AND y2 AND z2 ..., where others
+ * only recognize index operations when there are lists specified for each, such as x IN (x1,x2) AND y IN (y1,y2) AND z IN (z1,z2).
+ */
+ public String buildConjunctionClause(List outputParameters, ClauseDescription[] clauseDescriptions);
+
/** Obtain the maximum number of individual items that should be
* present in an IN clause. Exceeding this amount will potentially cause the query performance
* to drop.
Added: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java?rev=1195471&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java (added)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java Mon Oct 31 14:26:33 2011
@@ -0,0 +1,61 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.core.interfaces;
+
+import java.util.*;
+
+public class JoinClause implements ClauseDescription
+{
+ public static final String _rcsid = "@(#)$Id$";
+
+ protected String columnName;
+ protected String joinColumnName;
+
+ /** Construct */
+ public JoinClause(String columnName, String joinColumnName)
+ {
+ this.columnName = columnName;
+ this.joinColumnName = joinColumnName;
+ }
+
+ /** Get the column name */
+ public String getColumnName()
+ {
+ return columnName;
+ }
+
+ /** Get the operation string */
+ public String getOperation()
+ {
+ return "=";
+ }
+
+ /** Get the individual values */
+ public List getValues()
+ {
+ return null;
+ }
+
+ /** Get the join column name */
+ public String getJoinColumnName()
+ {
+ return joinColumnName;
+ }
+
+}
Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/MultiClause.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/MultiClause.java?rev=1195471&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/MultiClause.java (added)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/MultiClause.java Mon Oct 31 14:26:33 2011
@@ -0,0 +1,70 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.core.interfaces;
+
+import java.util.*;
+
+public class MultiClause implements ClauseDescription
+{
+ public static final String _rcsid = "@(#)$Id$";
+
+ protected String columnName;
+ protected List values;
+
+ /** Construct */
+ public MultiClause(String columnName, List values)
+ {
+ this.columnName = columnName;
+ this.values = values;
+ }
+
+ public MultiClause(String columnName, Object[] values)
+ {
+ this.columnName = columnName;
+ this.values = new ArrayList();
+ for (int i = 0 ; i < values.length ; i++)
+ {
+ this.values.add(values[i]);
+ }
+ }
+
+ /** Get the column name */
+ public String getColumnName()
+ {
+ return columnName;
+ }
+
+ /** Get the operation string */
+ public String getOperation()
+ {
+ return "=";
+ }
+
+ /** Get the individual values */
+ public List getValues()
+ {
+ return values;
+ }
+
+ /** Get the join column name */
+ public String getJoinColumnName()
+ {
+ return null;
+ }
+}
Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/MultiClause.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/MultiClause.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCheckClause.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCheckClause.java?rev=1195471&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCheckClause.java (added)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCheckClause.java Mon Oct 31 14:26:33 2011
@@ -0,0 +1,61 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.core.interfaces;
+
+import java.util.*;
+
+public class NullCheckClause implements ClauseDescription
+{
+ public static final String _rcsid = "@(#)$Id$";
+
+ protected String columnName;
+ protected boolean isNull;
+
+ /** Construct */
+ public NullCheckClause(String columnName, boolean isNull)
+ {
+ this.columnName = columnName;
+ this.isNull = isNull;
+ }
+
+ /** Get the column name */
+ public String getColumnName()
+ {
+ return columnName;
+ }
+
+ /** Get the operation string */
+ public String getOperation()
+ {
+ return isNull?" IS NULL":" IS NOT NULL";
+ }
+
+ /** Get the individual values */
+ public List getValues()
+ {
+ return null;
+ }
+
+ /** Get the join column name */
+ public String getJoinColumnName()
+ {
+ return null;
+ }
+
+}
Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCheckClause.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCheckClause.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/UnitaryClause.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/UnitaryClause.java?rev=1195471&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/UnitaryClause.java (added)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/UnitaryClause.java Mon Oct 31 14:26:33 2011
@@ -0,0 +1,68 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.core.interfaces;
+
+import java.util.*;
+
+public class UnitaryClause implements ClauseDescription
+{
+ public static final String _rcsid = "@(#)$Id$";
+
+ protected String columnName;
+ protected String operationName;
+ protected List values;
+
+ /** Construct */
+ public UnitaryClause(String columnName, String operationName, Object value)
+ {
+ this.columnName = columnName;
+ this.operationName = operationName;
+ this.values = new ArrayList();
+ this.values.add(value);
+ }
+
+ public UnitaryClause(String columnName, Object value)
+ {
+ this(columnName,"=",value);
+ }
+
+ /** Get the column name */
+ public String getColumnName()
+ {
+ return columnName;
+ }
+
+ /** Get the operation string */
+ public String getOperation()
+ {
+ return operationName;
+ }
+
+ /** Get the individual values */
+ public List getValues()
+ {
+ return values;
+ }
+
+ /** Get the join column name */
+ public String getJoinColumnName()
+ {
+ return null;
+ }
+}
Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/UnitaryClause.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/UnitaryClause.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java Mon Oct 31 14:26:33 2011
@@ -109,6 +109,10 @@ public class ManifoldCF
/** Database handle timeout property */
public static final String databaseHandleTimeoutProperty = "org.apache.manifoldcf.database.handletimeout";
+ // Database performance monitoring properties
+ /** Elapsed time a query can take before a warning is output to the log, in seconds */
+ public static final String databaseQueryMaxTimeProperty = "org.apache.manifoldcf.database.maxquerytime";
+
// Log configuration properties
/** Location of log configuration file */
public static final String logConfigFileProperty = "org.apache.manifoldcf.logconfigfile";
Modified: incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/Base.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/Base.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/Base.java (original)
+++ incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/Base.java Mon Oct 31 14:26:33 2011
@@ -95,6 +95,12 @@ public class Base
"<configuration>\n"+
" <property name=\"org.apache.manifoldcf.databaseimplementationclass\" value=\"org.apache.manifoldcf.core.database.DBInterfaceDerby\"/>\n" +
" <property name=\"org.apache.manifoldcf.derbydatabasepath\" value=\""+currentPathString.replaceAll("\\\\","/")+"\"/>\n" +
+ " <property name=\"org.apache.manifoldcf.database.maxquerytime\" value=\"30\"/>\n" +
+ " <property name=\"org.apache.manifoldcf.crawler.threads\" value=\"30\"/>\n" +
+ " <property name=\"org.apache.manifoldcf.crawler.expirethreads\" value=\"10\"/>\n" +
+ " <property name=\"org.apache.manifoldcf.crawler.cleanupthreads\" value=\"10\"/>\n" +
+ " <property name=\"org.apache.manifoldcf.crawler.deletethreads\" value=\"10\"/>\n" +
+ " <property name=\"org.apache.manifoldcf.database.maxhandles\" value=\"80\"/>\n" +
" <property name=\"org.apache.manifoldcf.logconfigfile\" value=\""+loggingFile.getAbsolutePath().replaceAll("\\\\","/")+"\"/>\n" +
"</configuration>\n");
Modified: incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseHSQLDB.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseHSQLDB.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseHSQLDB.java (original)
+++ incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseHSQLDB.java Mon Oct 31 14:26:33 2011
@@ -95,6 +95,12 @@ public class BaseHSQLDB
"<configuration>\n"+
" <property name=\"org.apache.manifoldcf.databaseimplementationclass\" value=\"org.apache.manifoldcf.core.database.DBInterfaceHSQLDB\"/>\n" +
" <property name=\"org.apache.manifoldcf.hsqldbdatabasepath\" value=\""+currentPathString.replaceAll("\\\\","/")+"\"/>\n" +
+ " <property name=\"org.apache.manifoldcf.database.maxquerytime\" value=\"30\"/>\n" +
+ " <property name=\"org.apache.manifoldcf.crawler.threads\" value=\"30\"/>\n" +
+ " <property name=\"org.apache.manifoldcf.crawler.expirethreads\" value=\"10\"/>\n" +
+ " <property name=\"org.apache.manifoldcf.crawler.cleanupthreads\" value=\"10\"/>\n" +
+ " <property name=\"org.apache.manifoldcf.crawler.deletethreads\" value=\"10\"/>\n" +
+ " <property name=\"org.apache.manifoldcf.database.maxhandles\" value=\"80\"/>\n" +
" <property name=\"org.apache.manifoldcf.logconfigfile\" value=\""+loggingFile.getAbsolutePath().replaceAll("\\\\","/")+"\"/>\n" +
"</configuration>\n");
Modified: incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BasePostgresql.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BasePostgresql.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BasePostgresql.java (original)
+++ incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BasePostgresql.java Mon Oct 31 14:26:33 2011
@@ -99,6 +99,7 @@ public class BasePostgresql
" <property name=\"org.apache.manifoldcf.databaseimplementationclass\" value=\"org.apache.manifoldcf.core.database.DBInterfacePostgreSQL\"/>\n" +
" <property name=\"org.apache.manifoldcf.database.name\" value=\"testdb\"/>\n" +
" <property name=\"org.apache.manifoldcf.database.username\" value=\"testuser\"/>\n" +
+ " <property name=\"org.apache.manifoldcf.database.maxquerytime\" value=\"30\"/>\n" +
" <property name=\"org.apache.manifoldcf.crawler.threads\" value=\"30\"/>\n" +
" <property name=\"org.apache.manifoldcf.crawler.expirethreads\" value=\"10\"/>\n" +
" <property name=\"org.apache.manifoldcf.crawler.cleanupthreads\" value=\"10\"/>\n" +
Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java Mon Oct 31 14:26:33 2011
@@ -258,6 +258,7 @@ public class AuthorityConnectionManager
StringSet cacheKeys = new StringSet(ssb);
while (true)
{
+ long sleepAmt = 0L;
try
{
ICacheHandle ch = cacheManager.enterCache(null,cacheKeys,getTransactionID());
@@ -271,9 +272,10 @@ public class AuthorityConnectionManager
boolean isNew = object.getIsNew();
// See whether the instance exists
ArrayList params = new ArrayList();
- params.add(object.getName());
+ String query = buildConjunctionClause(params,new ClauseDescription[]{
+ new UnitaryClause(nameField,object.getName())});
IResultSet set = performQuery("SELECT * FROM "+getTableName()+" WHERE "+
- nameField+"=? FOR UPDATE",params,null,null);
+ query+" FOR UPDATE",params,null,null);
HashMap values = new HashMap();
values.put(descriptionField,object.getDescription());
values.put(classNameField,object.getClassName());
@@ -290,8 +292,9 @@ public class AuthorityConnectionManager
isCreated = false;
// Update
params.clear();
- params.add(object.getName());
- performUpdate(values," WHERE "+nameField+"=?",params,null);
+ query = buildConjunctionClause(params,new ClauseDescription[]{
+ new UnitaryClause(nameField,object.getName())});
+ performUpdate(values," WHERE "+query,params,null);
}
else
{
@@ -333,19 +336,16 @@ public class AuthorityConnectionManager
// Is this a deadlock exception? If so, we want to try again.
if (e.getErrorCode() != ManifoldCFException.DATABASE_TRANSACTION_ABORT)
throw e;
- try
- {
- ManifoldCF.sleep((long)(random.nextDouble() * 60000.0 + 500.0));
- }
- catch (InterruptedException e2)
- {
- throw new ManifoldCFException(e2.getMessage(),e2,ManifoldCFException.INTERRUPTED);
- }
+ sleepAmt = getSleepAmt();
+ }
+ finally
+ {
+ sleepFor(sleepAmt);
}
}
}
- /** Delete a repository connection.
+ /** Delete an authority connection.
*@param name is the name of the connection to delete. If the
* name does not exist, no error is returned.
*/
@@ -370,8 +370,9 @@ public class AuthorityConnectionManager
throw new ManifoldCFException("Can't delete authority connection '"+name+"': existing repository connections refer to it");
ManifoldCF.noteConfigurationChange();
ArrayList params = new ArrayList();
- params.add(name);
- performDelete("WHERE "+nameField+"=?",params,null);
+ String query = buildConjunctionClause(params,new ClauseDescription[]{
+ new UnitaryClause(nameField,name)});
+ performDelete("WHERE "+query,params,null);
cacheManager.invalidateKeys(ch);
}
catch (ManifoldCFException e)
@@ -446,28 +447,23 @@ public class AuthorityConnectionManager
try
{
i = 0;
- StringBuilder sb = new StringBuilder();
ArrayList params = new ArrayList();
int j = 0;
- int maxIn = getMaxInClause();
+ int maxIn = maxClauseGetAuthorityConnectionsChunk();
while (i < connectionNames.length)
{
if (j == maxIn)
{
- getAuthorityConnectionsChunk(rval,returnIndex,sb.toString(),params);
- sb.setLength(0);
+ getAuthorityConnectionsChunk(rval,returnIndex,params);
params.clear();
j = 0;
}
- if (j > 0)
- sb.append(',');
- sb.append('?');
params.add(connectionNames[i]);
i++;
j++;
}
if (j > 0)
- getAuthorityConnectionsChunk(rval,returnIndex,sb.toString(),params);
+ getAuthorityConnectionsChunk(rval,returnIndex,params);
return rval;
}
catch (Error e)
@@ -486,18 +482,26 @@ public class AuthorityConnectionManager
}
}
+ /** Find the maximum number of clauses for getAuthorityConnectionsChunk.
+ */
+ protected int maxClauseGetAuthorityConnectionsChunk()
+ {
+ return findConjunctionClauseMax(new ClauseDescription[]{});
+ }
+
/** Read a chunk of authority connections.
*@param rval is the place to put the read policies.
*@param returnIndex is a map from the object id (resource id) and the rval index.
- *@param idList is the list of id's.
*@param params is the set of parameters.
*/
- protected void getAuthorityConnectionsChunk(AuthorityConnection[] rval, Map returnIndex, String idList, ArrayList params)
+ protected void getAuthorityConnectionsChunk(AuthorityConnection[] rval, Map returnIndex, ArrayList params)
throws ManifoldCFException
{
- IResultSet set;
- set = performQuery("SELECT * FROM "+getTableName()+" WHERE "+
- nameField+" IN ("+idList+")",params,null,null);
+ ArrayList list = new ArrayList();
+ String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new MultiClause(nameField,params)});
+ IResultSet set = performQuery("SELECT * FROM "+getTableName()+" WHERE "+
+ query,list,null,null);
int i = 0;
while (i < set.getRowCount())
{