You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/11/11 08:16:39 UTC
svn commit: r1638036 - in
/manifoldcf/branches/CONNECTORS-1100/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler:
interfaces/IReprioritizationTracker.java jobs/JobManager.java
system/SetPriorityThread.java
Author: kwright
Date: Tue Nov 11 07:16:39 2014
New Revision: 1638036
URL: http://svn.apache.org/r1638036
Log:
Set up SELECT-FOR_UPDATE structure for writing document priorities.
Modified:
manifoldcf/branches/CONNECTORS-1100/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java
manifoldcf/branches/CONNECTORS-1100/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
manifoldcf/branches/CONNECTORS-1100/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java
Modified: manifoldcf/branches/CONNECTORS-1100/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1100/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java?rev=1638036&r1=1638035&r2=1638036&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1100/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java (original)
+++ manifoldcf/branches/CONNECTORS-1100/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java Tue Nov 11 07:16:39 2014
@@ -85,7 +85,7 @@ public interface IReprioritizationTracke
*/
public void clearPreloadedValues();
- /** Get a bin value.
+ /** Get a bin value. Must be called INSIDE a transaction.
*@param binName is the bin name.
*@param weightedMinimumDepth is the minimum depth to use.
*@return the bin value.
Modified: manifoldcf/branches/CONNECTORS-1100/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1100/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1638036&r1=1638035&r2=1638036&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1100/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java (original)
+++ manifoldcf/branches/CONNECTORS-1100/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java Tue Nov 11 07:16:39 2014
@@ -2023,7 +2023,6 @@ public class JobManager implements IJobM
throws ManifoldCFException
{
// Note: This can be called at any time, even while worker and seeding threads are also reprioritizing documents.
- // Need to resolve the race condition somehow. ???
// Retry loop - in case we get a deadlock despite our best efforts
while (true)
@@ -2150,8 +2149,23 @@ public class JobManager implements IJobM
throw new ManifoldCFException("Assertion failure: duplicate document identifier jobid/hash detected!");
int index = x.intValue();
DocumentDescription dd = documentDescriptions[index];
- IPriorityCalculator priority = priorities[index];
- jobQueue.writeDocPriority(dd.getID(),priority);
+ // Query for the status
+ ArrayList list = new ArrayList();
+ String query = database.buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(jobQueue.idField,dd.getID())});
+ IResultSet set = database.performQuery("SELECT "+jobQueue.needPriorityField+" FROM "+jobQueue.getTableName()+" WHERE "+
+ query+" FOR UPDATE",list,null,null);
+ if (set.getRowCount() > 0)
+ {
+ IResultRow row = set.getRow(0);
+ // Grab the needPriority value
+ boolean needPriority = jobQueue.stringToNeedPriority((String)row.getValue(jobQueue.needPriorityField));
+ if (needPriority)
+ {
+ IPriorityCalculator priority = priorities[index];
+ jobQueue.writeDocPriority(dd.getID(),priority);
+ }
+ }
}
database.performCommit();
break;
Modified: manifoldcf/branches/CONNECTORS-1100/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1100/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java?rev=1638036&r1=1638035&r2=1638036&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1100/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java (original)
+++ manifoldcf/branches/CONNECTORS-1100/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java Tue Nov 11 07:16:39 2014
@@ -145,7 +145,7 @@ public class SetPriorityThread extends T
}
Logging.threads.debug("Done reprioritizing because no more documents to reprioritize");
- ManifoldCF.sleep(30000L);
+ ManifoldCF.sleep(5000L);
break;
}