You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/11/03 17:57:28 UTC
svn commit: r1636390 - in /manifoldcf/branches/release-1.7-branch: ./
framework/core/src/main/java/org/apache/manifoldcf/core/database/
framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/
framework/pull-agent/src/main/java/org/apache/ma...
Author: kwright
Date: Mon Nov 3 16:57:28 2014
New Revision: 1636390
URL: http://svn.apache.org/r1636390
Log:
Pull up fixes for CONNECTORS-1090 from dev_1x branch
Modified:
manifoldcf/branches/release-1.7-branch/ (props changed)
manifoldcf/branches/release-1.7-branch/CHANGES.txt
manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java
manifoldcf/branches/release-1.7-branch/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Propchange: manifoldcf/branches/release-1.7-branch/
------------------------------------------------------------------------------
Merged /manifoldcf/branches/dev_1x:r1635810,1636147,1636182
Merged /manifoldcf/trunk:r1635809,1636146,1636180
Modified: manifoldcf/branches/release-1.7-branch/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.7-branch/CHANGES.txt?rev=1636390&r1=1636389&r2=1636390&view=diff
==============================================================================
--- manifoldcf/branches/release-1.7-branch/CHANGES.txt (original)
+++ manifoldcf/branches/release-1.7-branch/CHANGES.txt Mon Nov 3 16:57:28 2014
@@ -2,8 +2,9 @@ ManifoldCF Change Log
$Id$
-CONNECTORS-1043: Fix list item URL.
-(Lalit Jangra, Karl Wright)
+CONNECTORS-1090: More PostgreSQL performance improvements.
+(Karl Wright)
+
======================= Release 1.7.1 =====================
Modified: manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java?rev=1636390&r1=1636389&r2=1636390&view=diff
==============================================================================
--- manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java (original)
+++ manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java Mon Nov 3 16:57:28 2014
@@ -611,7 +611,10 @@ public class DBInterfacePostgreSQL exten
return theException;
Throwable e = theException.getCause();
if (!(e instanceof java.sql.SQLException))
+ {
+ //e.printStackTrace();
return theException;
+ }
if (Logging.db.isDebugEnabled())
Logging.db.debug("Exception "+theException.getMessage()+" is possibly a transaction abort signal");
java.sql.SQLException sqlException = (java.sql.SQLException)e;
@@ -628,8 +631,14 @@ public class DBInterfacePostgreSQL exten
// one could make.)
if (sqlState != null && sqlState.equals("23505"))
return new ManifoldCFException(message,e,ManifoldCFException.DATABASE_TRANSACTION_ABORT);
+ // New Postgresql behavior (9.3): sometimes we don't get an exception thrown, but the transaction is dead nonetheless.
+ if (sqlState != null && sqlState.equals("25P02"))
+ return new ManifoldCFException(message,e,ManifoldCFException.DATABASE_TRANSACTION_ABORT);
+
if (Logging.db.isDebugEnabled())
Logging.db.debug("Exception "+theException.getMessage()+" is NOT a transaction abort signal");
+ //e.printStackTrace();
+ //System.err.println("sqlstate = "+sqlState);
return theException;
}
Modified: manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java?rev=1636390&r1=1636389&r2=1636390&view=diff
==============================================================================
--- manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java (original)
+++ manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java Mon Nov 3 16:57:28 2014
@@ -24,14 +24,21 @@ public class JoinClause implements Claus
{
public static final String _rcsid = "@(#)$Id$";
- protected String columnName;
- protected String joinColumnName;
+ protected final String columnName;
+ protected final String joinColumnName;
+ protected final String operation;
/** Construct */
public JoinClause(String columnName, String joinColumnName)
{
+ this(columnName,joinColumnName,"=");
+ }
+
+ public JoinClause(String columnName, String joinColumnName, String operation)
+ {
this.columnName = columnName;
this.joinColumnName = joinColumnName;
+ this.operation = "=";
}
/** Get the column name */
@@ -43,7 +50,7 @@ public class JoinClause implements Claus
/** Get the operation string */
public String getOperation()
{
- return "=";
+ return operation;
}
/** Get the individual values */
Modified: manifoldcf/branches/release-1.7-branch/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.7-branch/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1636390&r1=1636389&r2=1636390&view=diff
==============================================================================
--- manifoldcf/branches/release-1.7-branch/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java (original)
+++ manifoldcf/branches/release-1.7-branch/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java Mon Nov 3 16:57:28 2014
@@ -1522,18 +1522,17 @@ public class JobManager implements IJobM
sb.append("NOT EXISTS(SELECT 'x' FROM ").append(jobQueue.getTableName()).append(" t2 WHERE ")
.append(database.buildConjunctionClause(list,new ClauseDescription[]{
- new JoinClause("t2."+jobQueue.docHashField,"t0."+jobQueue.docHashField)})).append(" AND ")
- .append("t2.").append(jobQueue.statusField).append(" IN (?,?,?,?,?,?) AND ")
- .append("t2.").append(jobQueue.jobIDField).append("!=t0.").append(jobQueue.jobIDField)
+ new JoinClause("t2."+jobQueue.docHashField,"t0."+jobQueue.docHashField),
+ new MultiClause("t2."+jobQueue.statusField,new String[]{
+ jobQueue.statusToString(jobQueue.STATUS_ACTIVE),
+ jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY),
+ jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN),
+ jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY),
+ jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED),
+ jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED)}),
+ new JoinClause("t2."+jobQueue.jobIDField,"t0."+jobQueue.jobIDField,"!=")}))
.append(") ");
- list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVE));
- list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY));
- list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN));
- list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY));
- list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED));
- list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED));
-
sb.append(database.constructOffsetLimitClause(0,maxCount));
// The checktime is null field check is for backwards compatibility
@@ -1765,18 +1764,17 @@ public class JobManager implements IJobM
sb.append("NOT EXISTS(SELECT 'x' FROM ").append(jobQueue.getTableName()).append(" t2 WHERE ")
.append(database.buildConjunctionClause(list,new ClauseDescription[]{
- new JoinClause("t2."+jobQueue.docHashField,"t0."+jobQueue.docHashField)})).append(" AND ")
- .append("t2.").append(jobQueue.statusField).append(" IN (?,?,?,?,?,?) AND ")
- .append("t2.").append(jobQueue.jobIDField).append("!=t0.").append(jobQueue.jobIDField)
+ new JoinClause("t2."+jobQueue.docHashField,"t0."+jobQueue.docHashField),
+ new MultiClause("t2."+jobQueue.statusField,new String[]{
+ jobQueue.statusToString(jobQueue.STATUS_ACTIVE),
+ jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY),
+ jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN),
+ jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY),
+ jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED),
+ jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED)}),
+ new JoinClause("t2."+jobQueue.jobIDField,"t0."+jobQueue.jobIDField,"!=")}))
.append(") ");
- list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVE));
- list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY));
- list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN));
- list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY));
- list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED));
- list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED));
-
sb.append(database.constructOffsetLimitClause(0,maxCount));
// The checktime is null field check is for backwards compatibility
@@ -2304,16 +2302,16 @@ public class JobManager implements IJobM
sb.append("NOT EXISTS(SELECT 'x' FROM ").append(jobQueue.getTableName()).append(" t2 WHERE ")
.append(database.buildConjunctionClause(list,new ClauseDescription[]{
- new JoinClause("t2."+jobQueue.docHashField,"t0."+jobQueue.docHashField)})).append(" AND ")
- .append("t2.").append(jobQueue.statusField).append(" IN (?,?,?,?,?,?)").append(" AND ")
- .append("t2.").append(jobQueue.jobIDField).append("!=t0.").append(jobQueue.jobIDField).append(") ");
-
- list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVE));
- list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY));
- list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN));
- list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY));
- list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED));
- list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED));
+ new JoinClause("t2."+jobQueue.docHashField,"t0."+jobQueue.docHashField),
+ new MultiClause("t2."+jobQueue.statusField,new String[]{
+ jobQueue.statusToString(jobQueue.STATUS_ACTIVE),
+ jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY),
+ jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN),
+ jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY),
+ jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED),
+ jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED)}),
+ new JoinClause("t2."+jobQueue.jobIDField,"t0."+jobQueue.jobIDField,"!=")}))
+ .append(") ");
sb.append(database.constructOffsetLimitClause(0,n));
@@ -2836,7 +2834,7 @@ public class JobManager implements IJobM
.append(" t0 ").append(jobQueue.getGetNextDocumentsIndexHint()).append(" WHERE ");
sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{
- //new UnitaryClause("t0."+jobQueue.docPriorityField,">=",new Long(0L)),
+ new UnitaryClause("t0."+jobQueue.docPriorityField,"<",JobQueue.nullDocPriority), // Note: This is technically correct, but I need to confirm that it works OK for MySQL and HSQLDB
new MultiClause("t0."+jobQueue.statusField,new Object[]{
jobQueue.statusToString(JobQueue.STATUS_PENDING),
jobQueue.statusToString(JobQueue.STATUS_PENDINGPURGATORY)}),
@@ -2854,18 +2852,17 @@ public class JobManager implements IJobM
sb.append("NOT EXISTS(SELECT 'x' FROM ").append(jobQueue.getTableName()).append(" t2 WHERE ")
.append(database.buildConjunctionClause(list,new ClauseDescription[]{
- new JoinClause("t2."+jobQueue.docHashField,"t0."+jobQueue.docHashField)})).append(" AND ")
- .append("t2.").append(jobQueue.statusField).append(" IN (?,?,?,?,?,?) AND ")
- .append("t2.").append(jobQueue.jobIDField).append("!=t0.").append(jobQueue.jobIDField)
+ new JoinClause("t2."+jobQueue.docHashField,"t0."+jobQueue.docHashField),
+ new MultiClause("t2."+jobQueue.statusField,new String[]{
+ jobQueue.statusToString(jobQueue.STATUS_ACTIVE),
+ jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY),
+ jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN),
+ jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY),
+ jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED),
+ jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED)}),
+ new JoinClause("t2."+jobQueue.jobIDField,"t0."+jobQueue.jobIDField,"!=")}))
.append(") AND ");
- list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVE));
- list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY));
- list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN));
- list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY));
- list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED));
- list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED));
-
// Prerequisite event clause: AND NOT EXISTS(SELECT 'x' FROM prereqevents t3,events t4 WHERE t3.ownerid=t0.id AND t3.name=t4.name)
sb.append("NOT EXISTS(SELECT 'x' FROM ").append(jobQueue.prereqEventManager.getTableName()).append(" t3,").append(eventManager.getTableName()).append(" t4 WHERE t0.")
.append(jobQueue.idField).append("=t3.").append(jobQueue.prereqEventManager.ownerField).append(" AND t3.")