You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/09/17 03:29:08 UTC
svn commit: r1625452 - in /manifoldcf/branches/release-1.7-branch: ./
framework/core/src/main/java/org/apache/manifoldcf/core/database/
framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/
Author: kwright
Date: Wed Sep 17 01:29:07 2014
New Revision: 1625452
URL: http://svn.apache.org/r1625452
Log:
Pull up fix for CONNECTORS-1027 from dev_1x branch
Modified:
manifoldcf/branches/release-1.7-branch/ (props changed)
manifoldcf/branches/release-1.7-branch/CHANGES.txt
manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
manifoldcf/branches/release-1.7-branch/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Propchange: manifoldcf/branches/release-1.7-branch/
------------------------------------------------------------------------------
Merged /manifoldcf/branches/CONNECTORS-1027:r1624243-1624462
Merged /manifoldcf/branches/dev_1x:r1624465
Merged /manifoldcf/trunk:r1624464
Modified: manifoldcf/branches/release-1.7-branch/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.7-branch/CHANGES.txt?rev=1625452&r1=1625451&r2=1625452&view=diff
==============================================================================
--- manifoldcf/branches/release-1.7-branch/CHANGES.txt (original)
+++ manifoldcf/branches/release-1.7-branch/CHANGES.txt Wed Sep 17 01:29:07 2014
@@ -8,6 +8,10 @@ CONNECTORS-1031: Fix Zookeeper synchroni
against short tick time settings.
(Karl Wright)
+CONNECTORS-1027: Improve some general and PostgreSQL queries
+for large crawling sets.
+(Paul Boichat, Karl Wright)
+
CONNECTORS-1025: SharePoint connector should skip blocked files.
(Radek Sklenicka, Karl Wright)
Modified: manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java?rev=1625452&r1=1625451&r2=1625452&view=diff
==============================================================================
--- manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java (original)
+++ manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java Wed Sep 17 01:29:07 2014
@@ -1059,6 +1059,66 @@ public class DBInterfacePostgreSQL exten
return 25;
}
+ /* Calculate the number of values a particular clause can have, given the values for all the other clauses.
+ * For example, if in the expression x AND y AND z, x has 2 values and z has 1, find out how many values x can legally have
+ * when using the buildConjunctionClause() method below.
+ */
+ @Override
+ public int findConjunctionClauseMax(ClauseDescription[] otherClauseDescriptions)
+ {
+ // This implementation uses "OR"
+ return getMaxOrClause();
+ }
+
+ /* Construct a conjunction clause, e.g. x AND y AND z, where there is expected to be an index (x,y,z,...), and where x, y, or z
+ * can have multiple distinct values, The proper implementation of this method differs from database to database, because some databases
+ * only permit index operations when there are OR's between clauses, such as x1 AND y1 AND z1 OR x2 AND y2 AND z2 ..., where others
+ * only recognize index operations when there are lists specified for each, such as x IN (x1,x2) AND y IN (y1,y2) AND z IN (z1,z2).
+ */
+ @Override
+ public String buildConjunctionClause(List outputParameters, ClauseDescription[] clauseDescriptions)
+ {
+ // This implementation uses "OR" instead of "IN ()" for multiple values, since this generates better plans in Postgresql 9.x.
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0 ; i < clauseDescriptions.length ; i++)
+ {
+ ClauseDescription cd = clauseDescriptions[i];
+ if (i > 0)
+ sb.append(" AND ");
+ String columnName = cd.getColumnName();
+ List values = cd.getValues();
+ String operation = cd.getOperation();
+ String joinColumn = cd.getJoinColumnName();
+ if (values != null)
+ {
+ if (values.size() > 1)
+ {
+ sb.append(" (");
+ for (int j = 0 ; j < values.size() ; j++)
+ {
+ if (j > 0)
+ sb.append(" OR ");
+ sb.append(columnName).append(operation).append("?");
+ outputParameters.add(values.get(j));
+ }
+ sb.append(")");
+ }
+ else
+ {
+ sb.append(columnName).append(operation).append("?");
+ outputParameters.add(values.get(0));
+ }
+ }
+ else if (joinColumn != null)
+ {
+ sb.append(columnName).append(operation).append(joinColumn);
+ }
+ else
+ sb.append(columnName).append(operation);
+ }
+ return sb.toString();
+ }
+
/** For windowed report queries, e.g. maxActivity or maxBandwidth, obtain the maximum number of rows
* that can reasonably be expected to complete in an acceptable time.
*@return the maximum number of rows.
Modified: manifoldcf/branches/release-1.7-branch/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.7-branch/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1625452&r1=1625451&r2=1625452&view=diff
==============================================================================
--- manifoldcf/branches/release-1.7-branch/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java (original)
+++ manifoldcf/branches/release-1.7-branch/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java Wed Sep 17 01:29:07 2014
@@ -8564,10 +8564,11 @@ public class JobManager implements IJobM
StringBuilder sb = new StringBuilder("SELECT ");
ArrayList list = new ArrayList();
- sb.append(database.constructCountClause(JobQueue.docHashField)).append(" AS doccount")
- .append(" FROM ").append(jobQueue.getTableName()).append(" t1");
+ sb.append(database.constructCountClause("t2.x")).append(" AS doccount")
+ .append(" FROM (SELECT 'x' AS x FROM ").append(jobQueue.getTableName()).append(" t1");
addWhereClause(sb,list,whereClause,whereParams,false);
- sb.append(" ").append(database.constructOffsetLimitClause(0,maxCount+1,false));
+ sb.append(" ").append(database.constructOffsetLimitClause(0,maxCount+1,false))
+ .append(") t2");
IResultSet countResult = database.performQuery(sb.toString(),list,null,null);
if (countResult.getRowCount() > 0 && ((Long)countResult.getRow(0).getValue("doccount")).longValue() > maxCount)
{
@@ -8757,10 +8758,11 @@ public class JobManager implements IJobM
// Now, for each job, fire off a separate, limited, query for each count we care about
sb = new StringBuilder("SELECT ");
list.clear();
- sb.append(database.constructCountClause(JobQueue.docHashField)).append(" AS doccount")
- .append(" FROM ").append(jobQueue.getTableName()).append(" WHERE ");
+ sb.append(database.constructCountClause("t2.x")).append(" AS doccount")
+ .append(" FROM (SELECT 'x' AS x FROM ").append(jobQueue.getTableName()).append(" WHERE ");
sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{new UnitaryClause(JobQueue.jobIDField,jobID)}));
- sb.append(" ").append(database.constructOffsetLimitClause(0,maxCount+1,false));
+ sb.append(" ").append(database.constructOffsetLimitClause(0,maxCount+1,false))
+ .append(") t2");
IResultSet totalSet = database.performQuery(sb.toString(),list,null,null);
if (totalSet.getRowCount() > 0)
@@ -8780,12 +8782,13 @@ public class JobManager implements IJobM
sb = new StringBuilder("SELECT ");
list.clear();
- sb.append(database.constructCountClause(JobQueue.docHashField)).append(" AS doccount")
- .append(" FROM ").append(jobQueue.getTableName()).append(" WHERE ");
+ sb.append(database.constructCountClause("t2.x")).append(" AS doccount")
+ .append(" FROM (SELECT 'x' AS x FROM ").append(jobQueue.getTableName()).append(" WHERE ");
sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{new UnitaryClause(JobQueue.jobIDField,jobID)}));
sb.append(" AND ");
sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{buildOutstandingClause()}));
- sb.append(" ").append(database.constructOffsetLimitClause(0,maxCount+1,false));
+ sb.append(" ").append(database.constructOffsetLimitClause(0,maxCount+1,false))
+ .append(") t2");
IResultSet outstandingSet = database.performQuery(sb.toString(),list,null,null);
if (outstandingSet.getRowCount() > 0)
@@ -8805,12 +8808,13 @@ public class JobManager implements IJobM
sb = new StringBuilder("SELECT ");
list.clear();
- sb.append(database.constructCountClause(JobQueue.docHashField)).append(" AS doccount")
- .append(" FROM ").append(jobQueue.getTableName()).append(" WHERE ");
+ sb.append(database.constructCountClause("t2.x")).append(" AS doccount")
+ .append(" FROM (SELECT 'x' AS x FROM ").append(jobQueue.getTableName()).append(" WHERE ");
sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{new UnitaryClause(JobQueue.jobIDField,jobID)}));
sb.append(" AND ");
sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{buildProcessedClause()}));
- sb.append(" ").append(database.constructOffsetLimitClause(0,maxCount+1,false));
+ sb.append(" ").append(database.constructOffsetLimitClause(0,maxCount+1,false))
+ .append(") t2");
IResultSet processedSet = database.performQuery(sb.toString(),list,null,null);
if (processedSet.getRowCount() > 0)