You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2011/10/28 03:05:45 UTC
svn commit: r1190106 - in /incubator/lcf/branches/CONNECTORS-284: ./
framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/
framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ lib/
Author: kwright
Date: Fri Oct 28 01:05:44 2011
New Revision: 1190106
URL: http://svn.apache.org/viewvc?rev=1190106&view=rev
Log:
More improvements to queries to allow hsqldb to use them efficiently
Modified:
incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java
incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java
incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentDeleteStufferThread.java
incubator/lcf/branches/CONNECTORS-284/lib/hsqldb.jar
incubator/lcf/branches/CONNECTORS-284/mvn-bootstrap.bat
incubator/lcf/branches/CONNECTORS-284/mvn-bootstrap.sh
incubator/lcf/branches/CONNECTORS-284/pom.xml
Modified: incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java?rev=1190106&r1=1190105&r2=1190106&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java (original)
+++ incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java Fri Oct 28 01:05:44 2011
@@ -473,14 +473,14 @@ public class Carrydown extends org.apach
{
// Delete
StringBuilder sb = new StringBuilder("WHERE (");
- sb.append(query).append(") AND (").append(newField).append("=?)");
+ sb.append(query).append(") AND ").append(newField).append("=?");
ArrayList newList = (ArrayList)list.clone();
newList.add(statusToString(ISNEW_BASE));
performDelete(sb.toString(),newList,null);
// Restore new values
sb = new StringBuilder("WHERE (");
- sb.append(query).append(") AND (").append(newField).append("=? OR ").append(newField).append("=?)");
+ sb.append(query).append(") AND ").append(newField).append(" IN (?,?)");
list.add(statusToString(ISNEW_EXISTING));
list.add(statusToString(ISNEW_NEW));
HashMap map = new HashMap();
Modified: incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java?rev=1190106&r1=1190105&r2=1190106&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java (original)
+++ incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java Fri Oct 28 01:05:44 2011
@@ -187,7 +187,7 @@ public class IntrinsicLink extends org.a
map.put(newField,statusToString(LINKSTATUS_BASE));
list.add(statusToString(LINKSTATUS_NEW));
list.add(statusToString(LINKSTATUS_EXISTING));
- performUpdate(map,"WHERE "+newField+"=? OR "+newField+"=?",list,null);
+ performUpdate(map,"WHERE "+newField+" IN (?,?)",list,null);
}
/** Record a references from source to targets. These references will be marked as either "new" or "existing".
@@ -463,7 +463,7 @@ public class IntrinsicLink extends org.a
throws ManifoldCFException
{
StringBuilder sb = new StringBuilder("WHERE (");
- sb.append(query).append(") AND (").append(newField).append("=? OR ").append(newField).append("=?)");
+ sb.append(query).append(") AND ").append(newField).append(" IN (?,?)");
list.add(statusToString(LINKSTATUS_EXISTING));
list.add(statusToString(LINKSTATUS_NEW));
HashMap map = new HashMap();
Modified: incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java?rev=1190106&r1=1190105&r2=1190106&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java (original)
+++ incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java Fri Oct 28 01:05:44 2011
@@ -320,14 +320,14 @@ public class JobQueue extends org.apache
ArrayList list = new ArrayList();
list.add(statusToString(STATUS_ACTIVE));
list.add(statusToString(STATUS_ACTIVENEEDRESCAN));
- performUpdate(map,"WHERE "+statusField+"=? OR "+statusField+"=?",list,null);
+ performUpdate(map,"WHERE "+statusField+" IN (?,?)",list,null);
// Map ACTIVEPURGATORY to PENDINGPURGATORY
map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
list.clear();
list.add(statusToString(STATUS_ACTIVEPURGATORY));
list.add(statusToString(STATUS_ACTIVENEEDRESCANPURGATORY));
- performUpdate(map,"WHERE "+statusField+"=? OR "+statusField+"=?",list,null);
+ performUpdate(map,"WHERE "+statusField+" IN (?,?)",list,null);
// Map BEINGDELETED to ELIGIBLEFORDELETE
map.put(statusField,statusToString(STATUS_ELIGIBLEFORDELETE));
@@ -388,14 +388,14 @@ public class JobQueue extends org.apache
ArrayList list = new ArrayList();
list.add(statusToString(STATUS_ACTIVE));
list.add(statusToString(STATUS_ACTIVENEEDRESCAN));
- performUpdate(map,"WHERE "+statusField+"=? OR "+statusField+"=?",list,null);
+ performUpdate(map,"WHERE "+statusField+" IN (?,?)",list,null);
// Map ACTIVEPURGATORY to PENDINGPURGATORY
map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
list.clear();
list.add(statusToString(STATUS_ACTIVEPURGATORY));
list.add(statusToString(STATUS_ACTIVENEEDRESCANPURGATORY));
- performUpdate(map,"WHERE "+statusField+"=? OR "+statusField+"=?",list,null);
+ performUpdate(map,"WHERE "+statusField+" IN (?,?)",list,null);
}
/** Reset doc delete worker status.
Modified: incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java?rev=1190106&r1=1190105&r2=1190106&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java (original)
+++ incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java Fri Oct 28 01:05:44 2011
@@ -766,7 +766,7 @@ public class Jobs extends org.apache.man
list.add(statusToString(STATUS_STARTINGUP));
list.add(statusToString(STATUS_ABORTINGSTARTINGUP));
map.put(statusField,statusToString(STATUS_READYFORSTARTUP));
- performUpdate(map,"WHERE "+statusField+"=? OR "+statusField+"=?",list,invKey);
+ performUpdate(map,"WHERE "+statusField+" IN (?,?)",list,invKey);
// Aborting starting up for restart state goes to ABORTINGFORRESTART
list.clear();
Modified: incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentDeleteStufferThread.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentDeleteStufferThread.java?rev=1190106&r1=1190105&r2=1190106&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentDeleteStufferThread.java (original)
+++ incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/DocumentDeleteStufferThread.java Fri Oct 28 01:05:44 2011
@@ -88,7 +88,7 @@ public class DocumentDeleteStufferThread
// Wait until the delete queue is "empty" (meaning that some delete threads
// can run out of work if we don't act).
- if (documentDeleteQueue.checkIfEmpty(n) == false)
+ if (documentDeleteQueue.checkIfEmpty(n * 3) == false)
{
ManifoldCF.sleep(100L);
continue;
Modified: incubator/lcf/branches/CONNECTORS-284/lib/hsqldb.jar
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-284/lib/hsqldb.jar?rev=1190106&r1=1190105&r2=1190106&view=diff
==============================================================================
Binary files - no diff available.
Modified: incubator/lcf/branches/CONNECTORS-284/mvn-bootstrap.bat
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-284/mvn-bootstrap.bat?rev=1190106&r1=1190105&r2=1190106&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-284/mvn-bootstrap.bat (original)
+++ incubator/lcf/branches/CONNECTORS-284/mvn-bootstrap.bat Fri Oct 28 01:05:44 2011
@@ -17,7 +17,7 @@ rem limitations under the License.
ant download-dependencies
mvn install:install-file -Dfile=lib/jdbcpool-0.99.jar -DgroupId=com.bitmechanic -DartifactId=jdbcpool -Dversion=0.99 -Dpackaging=jar
-mvn install:install-file -Dfile=lib/hsqldb.jar -DgroupId=org.hsqldb -DartifactId=hsqldb -Dversion=2.2.5.6-9-2011 -Dpackaging=jar
+mvn install:install-file -Dfile=lib/hsqldb.jar -DgroupId=org.hsqldb -DartifactId=hsqldb -Dversion=2.2.5.10-27-2011 -Dpackaging=jar
mvn install:install-file -Dfile=lib/commons-httpclient-mcf.jar -DgroupId=commons-httpclient -DartifactId=commons-httpclient-mcf -Dversion=3.1 -Dpackaging=jar
mvn install:install-file -Dfile=lib/xercesImpl-mcf.jar -DgroupId=xerces -DartifactId=xercesImpl-mcf -Dversion=2.9.1 -Dpackaging=jar
Modified: incubator/lcf/branches/CONNECTORS-284/mvn-bootstrap.sh
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-284/mvn-bootstrap.sh?rev=1190106&r1=1190105&r2=1190106&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-284/mvn-bootstrap.sh (original)
+++ incubator/lcf/branches/CONNECTORS-284/mvn-bootstrap.sh Fri Oct 28 01:05:44 2011
@@ -24,7 +24,7 @@
ant download-dependencies
mvn install:install-file -Dfile=lib/jdbcpool-0.99.jar -DgroupId=com.bitmechanic -DartifactId=jdbcpool -Dversion=0.99 -Dpackaging=jar
-mvn install:install-file -Dfile=lib/hsqldb.jar -DgroupId=org.hsqldb -DartifactId=hsqldb -Dversion=2.2.5.6-9-2011 -Dpackaging=jar
+mvn install:install-file -Dfile=lib/hsqldb.jar -DgroupId=org.hsqldb -DartifactId=hsqldb -Dversion=2.2.5.10-27-2011 -Dpackaging=jar
mvn install:install-file -Dfile=lib/commons-httpclient-mcf.jar -DgroupId=commons-httpclient -DartifactId=commons-httpclient-mcf -Dversion=3.1 -Dpackaging=jar
mvn install:install-file -Dfile=lib/xercesImpl-mcf.jar -DgroupId=xerces -DartifactId=xercesImpl-mcf -Dversion=2.9.1 -Dpackaging=jar
Modified: incubator/lcf/branches/CONNECTORS-284/pom.xml
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-284/pom.xml?rev=1190106&r1=1190105&r2=1190106&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-284/pom.xml (original)
+++ incubator/lcf/branches/CONNECTORS-284/pom.xml Fri Oct 28 01:05:44 2011
@@ -40,7 +40,7 @@
<!-- dependency versions -->
<junit.version>4.8.2</junit.version>
<postgresql.version>8.4-702.jdbc4</postgresql.version>
- <hsqldb.version>2.2.5.6-9-2011</hsqldb.version>
+ <hsqldb.version>2.2.5.10-27-2011</hsqldb.version>
<derby.version>10.7.1.1</derby.version>
<jetty.version>6.1.26</jetty.version>
</properties>