You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2011/10/28 13:30:00 UTC
svn commit: r1190252 - in
/incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs:
JobManager.java JobQueue.java
Author: kwright
Date: Fri Oct 28 11:30:00 2011
New Revision: 1190252
URL: http://svn.apache.org/viewvc?rev=1190252&view=rev
Log:
More alignment of indexes with queries
Modified:
incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
Modified: incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1190252&r1=1190251&r2=1190252&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java (original)
+++ incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java Fri Oct 28 11:30:00 2011
@@ -865,9 +865,11 @@ public class JobManager implements IJobM
" AND (t0."+jobQueue.checkTimeField+" IS NULL OR t0."+jobQueue.checkTimeField+"<=?) "+
" AND EXISTS(SELECT 'x' FROM "+jobs.getTableName()+" t1 WHERE t0."+jobQueue.jobIDField+"=t1."+jobs.idField+
" AND t1."+jobs.statusField+"=?"+
- ") AND NOT EXISTS(SELECT 'x' FROM "+jobQueue.getTableName()+" t2 WHERE t0."+jobQueue.docHashField+"=t2."+
- jobQueue.docHashField+" AND t0."+jobQueue.jobIDField+"!=t2."+jobQueue.jobIDField+
- " AND t2."+jobQueue.statusField+" IN (?,?,?,?,?,?)) "+database.constructOffsetLimitClause(0,maxCount),
+ ") AND NOT EXISTS(SELECT 'x' FROM "+jobQueue.getTableName()+" t2 WHERE "+
+ "t2."+jobQueue.docHashField+"=t0."+jobQueue.docHashField+" AND "+
+ "t2."+jobQueue.statusField+" IN (?,?,?,?,?,?)"+" AND "+
+ "t2."+jobQueue.jobIDField+"!=t0."+jobQueue.jobIDField+
+ ") "+database.constructOffsetLimitClause(0,maxCount),
list,null,null,maxCount,null);
if (Logging.perf.isDebugEnabled())
@@ -1097,9 +1099,11 @@ public class JobManager implements IJobM
" AND t0."+jobQueue.checkTimeField+"<=? "+
" AND EXISTS(SELECT 'x' FROM "+jobs.getTableName()+" t1 WHERE t0."+jobQueue.jobIDField+"=t1."+jobs.idField+
" AND t1."+jobs.statusField+"=?"+
- ") AND NOT EXISTS(SELECT 'x' FROM "+jobQueue.getTableName()+" t2 WHERE t0."+jobQueue.docHashField+"=t2."+
- jobQueue.docHashField+" AND t0."+jobQueue.jobIDField+"!=t2."+jobQueue.jobIDField+
- " AND t2."+jobQueue.statusField+" IN (?,?,?,?,?,?)) "+database.constructOffsetLimitClause(0,maxCount),
+ ") AND NOT EXISTS(SELECT 'x' FROM "+jobQueue.getTableName()+" t2 WHERE "+
+ "t2."+jobQueue.docHashField+"=t0."+jobQueue.docHashField+" AND "+
+ "t2."+jobQueue.statusField+" IN (?,?,?,?,?,?)"+" AND "+
+ "t2."+jobQueue.jobIDField+"!=t0."+jobQueue.jobIDField+
+ ") "+database.constructOffsetLimitClause(0,maxCount),
list,null,null,maxCount,null);
if (Logging.perf.isDebugEnabled())
@@ -1627,11 +1631,11 @@ public class JobManager implements IJobM
.append("t0.").append(jobQueue.checkTimeField).append("<=? AND ")
.append("t0.").append(jobQueue.jobIDField).append("=t1.").append(jobs.idField).append(" AND t1.")
.append(jobs.statusField).append(" IN (?,?) AND ");
- sb.append("NOT EXISTS(SELECT 'x' FROM ").append(jobQueue.getTableName()).append(" t2 WHERE t0.")
- .append(jobQueue.docHashField).append("=t2.").append(jobQueue.docHashField).append(" AND t0.")
- .append(jobQueue.jobIDField).append("!=t2.").append(jobQueue.jobIDField).append(" AND t2.")
- .append(jobQueue.statusField).append(" IN (?,?,?,?,?,?))");
- sb.append(" ").append(database.constructOffsetLimitClause(0,n));
+ sb.append("NOT EXISTS(SELECT 'x' FROM ").append(jobQueue.getTableName()).append(" t2 WHERE ")
+ .append("t2.").append(jobQueue.docHashField).append("=t0.").append(jobQueue.docHashField).append(" AND ")
+ .append("t2.").append(jobQueue.statusField).append(" IN (?,?,?,?,?,?)").append(" AND ")
+ .append("t2.").append(jobQueue.jobIDField).append("!=t0.").append(jobQueue.jobIDField);
+ sb.append(") ").append(database.constructOffsetLimitClause(0,n));
// Analyze jobqueue tables unconditionally, since it's become much more sensitive in 8.3 than it used to be.
jobQueue.unconditionallyAnalyzeTables();
@@ -2137,10 +2141,11 @@ public class JobManager implements IJobM
.append("t0.").append(jobQueue.checkActionField).append("=? AND ")
.append("t0.").append(jobQueue.checkTimeField).append("<=? AND ");
- sb.append("NOT EXISTS(SELECT 'x' FROM ").append(jobQueue.getTableName()).append(" t2 WHERE t0.")
- .append(jobQueue.docHashField).append("=t2.").append(jobQueue.docHashField).append(" AND t0.")
- .append(jobQueue.jobIDField).append("!=t2.").append(jobQueue.jobIDField).append(" AND t2.")
- .append(jobQueue.statusField).append(" IN (?,?,?,?,?,?)) AND ");
+ sb.append("NOT EXISTS(SELECT 'x' FROM ").append(jobQueue.getTableName()).append(" t2 WHERE ")
+ .append("t2.").append(jobQueue.docHashField).append("=t0.").append(jobQueue.docHashField).append(" AND ")
+ .append("t2.").append(jobQueue.statusField).append(" IN (?,?,?,?,?,?)").append(" AND ")
+ .append("t2.").append(jobQueue.jobIDField).append("!=t0.").append(jobQueue.jobIDField)
+ .append(") AND ");
// Prerequisite event clause: AND NOT EXISTS(SELECT 'x' FROM prereqevents t3,events t4 WHERE t3.ownerid=t0.id AND t3.name=t4.name)
sb.append("NOT EXISTS(SELECT 'x' FROM ").append(jobQueue.prereqEventManager.getTableName()).append(" t3,").append(eventManager.getTableName()).append(" t4 WHERE t0.")
@@ -3155,12 +3160,12 @@ public class JobManager implements IJobM
String[] docPrereqs = reorderedDocumentPrerequisites[z];
ArrayList list = new ArrayList();
- list.add(jobID);
list.add(docIDHash);
+ list.add(jobID);
IResultSet set = database.performQuery("SELECT "+jobQueue.idField+","+jobQueue.statusField+","+
jobQueue.checkTimeField+" FROM "+jobQueue.getTableName()+
- " WHERE "+jobQueue.jobIDField+"=? AND "+jobQueue.docHashField+"=? FOR UPDATE",list,null,null);
+ " WHERE "+jobQueue.docHashField+"=? AND "+jobQueue.jobIDField+"=? FOR UPDATE",list,null,null);
boolean priorityUsed;
long executeTime = overrideSchedule?0L:-1L;
@@ -3743,12 +3748,12 @@ public class JobManager implements IJobM
String docIDHash = reorderedDocIDHashes[z];
ArrayList list = new ArrayList();
- list.add(jobID);
list.add(docIDHash);
+ list.add(jobID);
IResultSet set = database.performQuery("SELECT "+jobQueue.idField+","+jobQueue.statusField+","+
jobQueue.checkTimeField+" FROM "+jobQueue.getTableName()+
- " WHERE "+jobQueue.jobIDField+"=? AND "+jobQueue.docHashField+"=? FOR UPDATE",list,null,null);
+ " WHERE "+jobQueue.docHashField+"=? AND "+jobQueue.jobIDField+"=? FOR UPDATE",list,null,null);
boolean priorityUsed;
@@ -4049,7 +4054,7 @@ public class JobManager implements IJobM
jobQueue.getTableName()+" t0 WHERE "+
"t0."+jobQueue.jobIDField+"=? AND "+
"EXISTS(SELECT 'x' FROM "+carryDown.getTableName()+" t1 WHERE "+
- "t0."+jobQueue.jobIDField+"=t1."+jobQueue.jobIDField+" AND "+
+ "t1."+carryDown.jobIDField+"=t0."+jobQueue.jobIDField+" AND "+
carryDown.parentIDHashField+" IN ("+queryPart+") AND "+
"t1."+carryDown.childIDHashField+"=t0."+jobQueue.docHashField+" AND "+
"t1."+carryDown.newField+"=?)";
Modified: incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java?rev=1190252&r1=1190251&r2=1190252&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java (original)
+++ incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java Fri Oct 28 11:30:00 2011
@@ -204,7 +204,7 @@ public class JobQueue extends org.apache
IndexDescription uniqueIndex = new IndexDescription(true,new String[]{docHashField,jobIDField});
IndexDescription jobStatusIndex = new IndexDescription(false,new String[]{jobIDField,statusField});
IndexDescription jobSeedIndex = new IndexDescription(false,new String[]{jobIDField,isSeedField});
- IndexDescription jobHashStatusIndex = new IndexDescription(false,new String[]{jobIDField,docHashField,statusField});
+ IndexDescription jobHashStatusIndex = new IndexDescription(false,new String[]{docHashField,statusField});
IndexDescription actionTimeStatusIndex = new IndexDescription(false,new String[]{statusField,checkActionField,checkTimeField});
IndexDescription prioritysetStatusIndex = new IndexDescription(false,new String[]{prioritySetField,statusField,checkActionField});
IndexDescription docpriorityIndex = new IndexDescription(false,new String[]{docPriorityField});