You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/11/18 15:30:38 UTC
svn commit: r1640350 [1/2] - in /manifoldcf/branches/dev_1x: ./ framework/
framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/
framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/
framework/pull-agent/src/main...
Author: kwright
Date: Tue Nov 18 14:30:38 2014
New Revision: 1640350
URL: http://svn.apache.org/r1640350
Log:
Pull up fix for CONNECTORS-1100 from trunk, plus upgrade from the CONNECTORS-1100-1x branch
Modified:
manifoldcf/branches/dev_1x/ (props changed)
manifoldcf/branches/dev_1x/CHANGES.txt
manifoldcf/branches/dev_1x/framework/ (props changed)
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobResetThread.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingActivity.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java
manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
Merged /manifoldcf/branches/CONNECTORS-1100:r1637693-1640317
Merged /manifoldcf/trunk:r1640319
Modified: manifoldcf/branches/dev_1x/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/CHANGES.txt?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/CHANGES.txt (original)
+++ manifoldcf/branches/dev_1x/CHANGES.txt Tue Nov 18 14:30:38 2014
@@ -3,6 +3,10 @@ $Id$
======================= 1.8-dev =====================
+CONNECTORS-1100: Change how reprioritization is done to improve
+overall usability at large scale.
+(Aeham Abushwashi, Karl Wright)
+
CONNECTORS-1106: Missing state handling for abort case.
(Kamil Żyta, Karl Wright)
Propchange: manifoldcf/branches/dev_1x/framework/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk/framework:r1640319
Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java Tue Nov 18 14:30:38 2014
@@ -215,23 +215,29 @@ public interface IJobManager
// These methods support the "set doc priority" thread
+ /** Clear all document priorities, in preparation for reprioritization of all previously-prioritized documents.
+ * This method is called to start the dynamic reprioritization cycle, which follows this
+ * method with explicit prioritization of all documents, piece-meal, using getNextNotYetProcessedReprioritizationDocuments(),
+ * and writeDocumentPriorities().
+ */
+ public void clearAllDocumentPriorities()
+ throws ManifoldCFException;
+
/** Get a list of not-yet-processed documents to reprioritize. Documents in all jobs will be
* returned by this method. Up to n document descriptions will be returned.
- *@param currentTime is the current time stamp for this prioritization pass. Avoid
- * picking up any documents that are labeled with this timestamp or after.
+ *@param processID is the process that requests the reprioritization documents.
*@param n is the maximum number of document descriptions desired.
*@return the document descriptions.
*/
- public DocumentDescription[] getNextNotYetProcessedReprioritizationDocuments(long currentTime, int n)
+ public DocumentDescription[] getNextNotYetProcessedReprioritizationDocuments(String processID, int n)
throws ManifoldCFException;
/** Save a set of document priorities. In the case where a document was eligible to have its
* priority set, but it no longer is eligible, then the provided priority will not be written.
- *@param currentTime is the time in milliseconds since epoch.
*@param descriptions are the document descriptions.
*@param priorities are the desired priorities.
*/
- public void writeDocumentPriorities(long currentTime, DocumentDescription[] descriptions, IPriorityCalculator[] priorities)
+ public void writeDocumentPriorities(DocumentDescription[] descriptions, IPriorityCalculator[] priorities)
throws ManifoldCFException;
// This method supports the "expiration" thread
@@ -414,7 +420,7 @@ public interface IJobManager
*@param documentDescriptions is the set of description objects for the documents that have had their parent carrydown information changed.
*@param docPriorities are the document priorities to assign to the documents, if needed.
*/
- public void carrydownChangeDocumentMultiple(DocumentDescription[] documentDescriptions, long currentTime, IPriorityCalculator[] docPriorities)
+ public void carrydownChangeDocumentMultiple(DocumentDescription[] documentDescriptions, IPriorityCalculator[] docPriorities)
throws ManifoldCFException;
/** Requeue a document because of carrydown changes.
@@ -423,7 +429,7 @@ public interface IJobManager
*@param documentDescription is the description object for the document that has had its parent carrydown information changed.
*@param docPriority is the document priority to assign to the document, if needed.
*/
- public void carrydownChangeDocument(DocumentDescription documentDescription, long currentTime, IPriorityCalculator docPriority)
+ public void carrydownChangeDocument(DocumentDescription documentDescription, IPriorityCalculator docPriority)
throws ManifoldCFException;
/** Requeue a document for further processing in the future.
@@ -558,14 +564,13 @@ public interface IJobManager
*@param docIDs are the local document identifiers.
*@param overrideSchedule is true if any existing document schedule should be overridden.
*@param hopcountMethod is either accurate, nodelete, or neverdelete.
- *@param currentTime is the current time in milliseconds since epoch.
*@param documentPriorities are the document priorities corresponding to the document identifiers.
*@param prereqEventNames are the events that must be completed before each document can be processed.
*/
public void addDocumentsInitial(String processID,
Long jobID, String[] legalLinkTypes,
String[] docIDHashes, String[] docIDs, boolean overrideSchedule,
- int hopcountMethod, long currentTime, IPriorityCalculator[] documentPriorities,
+ int hopcountMethod, IPriorityCalculator[] documentPriorities,
String[][] prereqEventNames)
throws ManifoldCFException;
@@ -650,7 +655,6 @@ public interface IJobManager
*@param dataNames are the names of the data to carry down to the child from this parent.
*@param dataValues are the values to carry down to the child from this parent, corresponding to dataNames above. If CharacterInput objects are passed in here,
* it is the caller's responsibility to clean these up.
- *@param currentTime is the time in milliseconds since epoch that will be recorded for this operation.
*@param priority is the desired document priority for the document.
*@param prereqEventNames are the events that must be completed before the document can be processed.
*/
@@ -660,7 +664,7 @@ public interface IJobManager
String parentIdentifierHash,
String relationshipType,
int hopcountMethod, String[] dataNames, Object[][] dataValues,
- long currentTime, IPriorityCalculator priority, String[] prereqEventNames)
+ IPriorityCalculator priority, String[] prereqEventNames)
throws ManifoldCFException;
/** Add documents to the queue in bulk.
@@ -680,7 +684,6 @@ public interface IJobManager
*@param dataNames are the names of the data to carry down to the child from this parent.
*@param dataValues are the values to carry down to the child from this parent, corresponding to dataNames above. If CharacterInput objects are passed in here,
* it is the caller's responsibility to clean these up.
- *@param currentTime is the time in milliseconds since epoch that will be recorded for this operation.
*@param priorities are the desired document priorities for the documents.
*@param prereqEventNames are the events that must be completed before each document can be processed.
*/
@@ -690,7 +693,7 @@ public interface IJobManager
String parentIdentifierHash,
String relationshipType,
int hopcountMethod, String[][] dataNames, Object[][][] dataValues,
- long currentTime, IPriorityCalculator[] priorities,
+ IPriorityCalculator[] priorities,
String[][] prereqEventNames)
throws ManifoldCFException;
Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java Tue Nov 18 14:30:38 2014
@@ -31,21 +31,13 @@ public interface IReprioritizationTracke
public static final String _rcsid = "@(#)$Id$";
/** Start a reprioritization activity.
- *@param prioritizationTime is the timestamp of the prioritization.
*@param processID is the process ID of the process performing/waiting for the prioritization
* to complete.
*@param reproID is the reprocessing thread ID
*/
- public void startReprioritization(long prioritizationTime, String processID, String reproID)
+ public void startReprioritization(String processID, String reproID)
throws ManifoldCFException;
- /** Retrieve the current reprioritization time stamp. This should be obtained before
- * performing any prioritization steps.
- *@return the current prioritization timestamp, or null if no prioritization is in effect.
- */
- public Long checkReprioritizationInProgress()
- throws ManifoldCFException;
-
/** Complete a reprioritization activity. Prioritization will be marked as complete
* only if the processID matches the one that started the current reprioritization.
*@param processID is the process ID of the process completing the prioritization.
@@ -93,7 +85,7 @@ public interface IReprioritizationTracke
*/
public void clearPreloadedValues();
- /** Get a bin value.
+ /** Get a bin value. Must be called INSIDE a transaction.
*@param binName is the bin name.
*@param weightedMinimumDepth is the minimum depth to use.
*@return the bin value.
Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java Tue Nov 18 14:30:38 2014
@@ -33,6 +33,7 @@ public class JobManager implements IJobM
public static final String _rcsid = "@(#)$Id: JobManager.java 998576 2010-09-19 01:11:02Z kwright $";
protected static final String stufferLock = "_STUFFER_";
+ protected static final String reprioritizationLock = "_REPRIORITIZER_";
protected static final String deleteStufferLock = "_DELETESTUFFER_";
protected static final String expireStufferLock = "_EXPIRESTUFFER_";
protected static final String cleanStufferLock = "_CLEANSTUFFER_";
@@ -2061,92 +2062,165 @@ public class JobManager implements IJobM
// These methods support the reprioritization thread.
+ /** Clear all document priorities, in preparation for reprioritization of all previously-prioritized documents.
+ * This method is called to start the dynamic reprioritization cycle, which follows this
+ * method with explicit prioritization of all documents, piece-meal, using getNextNotYetProcessedReprioritizationDocuments(),
+ * and writeDocumentPriorities().
+ */
+ public void clearAllDocumentPriorities()
+ throws ManifoldCFException
+ {
+ // Note: This can be called at any time, even while worker and seeding threads are also reprioritizing documents.
+
+ // Retry loop - in case we get a deadlock despite our best efforts
+ while (true)
+ {
+ long sleepAmt = 0L;
+
+ // Start the transaction now
+ database.beginTransaction();
+ try
+ {
+ jobQueue.clearAllDocPriorities();
+ database.performCommit();
+ break;
+ }
+ catch (ManifoldCFException e)
+ {
+ database.signalRollback();
+ if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
+ {
+ if (Logging.perf.isDebugEnabled())
+ Logging.perf.debug("Aborted transaction clearing doc priorities for reprioritization: "+e.getMessage());
+ sleepAmt = getRandomAmount();
+ continue;
+ }
+ throw e;
+ }
+ catch (Error e)
+ {
+ database.signalRollback();
+ throw e;
+ }
+ finally
+ {
+ database.endTransaction();
+ sleepFor(sleepAmt);
+ }
+ }
+ }
+
/** Get a list of not-yet-processed documents to reprioritize. Documents in all jobs will be
* returned by this method. Up to n document descriptions will be returned.
- *@param currentTime is the current time stamp for this prioritization pass. Avoid
- * picking up any documents that are labeled with this timestamp or after.
+ *@param processID is the process that requests the reprioritization documents.
*@param n is the maximum number of document descriptions desired.
*@return the document descriptions.
*/
@Override
- public DocumentDescription[] getNextNotYetProcessedReprioritizationDocuments(long currentTime, int n)
+ public DocumentDescription[] getNextNotYetProcessedReprioritizationDocuments(String processID, int n)
throws ManifoldCFException
{
- StringBuilder sb = new StringBuilder("SELECT ");
- ArrayList list = new ArrayList();
+ // Retry loop - in case we get a deadlock despite our best efforts
+ while (true)
+ {
+ long sleepAmt = 0L;
- // This query MUST return only documents that are in a pending state which belong to an active job!!!
+ // Write lock insures that only one thread cluster-wide can be doing this at a given time, so FOR UPDATE is unneeded.
+ lockManager.enterWriteLock(reprioritizationLock);
+ try
+ {
+ // Start the transaction now
+ database.beginTransaction();
+ try
+ {
- sb.append(jobQueue.idField).append(",")
- .append(jobQueue.docHashField).append(",")
- .append(jobQueue.docIDField).append(",")
- .append(jobQueue.jobIDField)
- .append(" FROM ").append(jobQueue.getTableName()).append(" t0 WHERE ")
- .append(database.buildConjunctionClause(list,new ClauseDescription[]{
- new MultiClause(jobQueue.statusField,new Object[]{
- JobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVED),
- JobQueue.statusToString(jobQueue.STATUS_PENDING),
- JobQueue.statusToString(jobQueue.STATUS_PENDINGPURGATORY)}),
- new UnitaryClause(jobQueue.prioritySetField,"<",new Long(currentTime))}));
-
- sb.append(" AND ")
- .append(jobQueue.checkActionField).append("=?");
- list.add(jobQueue.actionToString(JobQueue.ACTION_RESCAN));
+ StringBuilder sb = new StringBuilder("SELECT ");
+ ArrayList list = new ArrayList();
- // Per CONNECTORS-290, we need to be leaving priorities blank for jobs that aren't using them,
- // so this will be changed to not include jobs where the priorities have been bashed to null.
- //
- // I've included ALL states that might have non-null doc priorities. This includes states
- // corresponding to uninstalled connectors, since there is no transition that cleans out the
- // document priorities in these states. The time during which a connector is uninstalled is
- // expected to be short, because typically this state is the result of an installation procedure
- // rather than willful action on the part of a user.
-
- sb.append(" AND EXISTS(SELECT 'x' FROM ").append(jobs.getTableName()).append(" t1 WHERE ")
- .append(database.buildConjunctionClause(list,new ClauseDescription[]{
- new MultiClause("t1."+jobs.statusField,new Object[]{
- Jobs.statusToString(Jobs.STATUS_STARTINGUP),
- Jobs.statusToString(Jobs.STATUS_STARTINGUPMINIMAL),
- Jobs.statusToString(Jobs.STATUS_ACTIVE),
- Jobs.statusToString(Jobs.STATUS_ACTIVESEEDING),
- Jobs.statusToString(Jobs.STATUS_ACTIVE_UNINSTALLED),
- Jobs.statusToString(Jobs.STATUS_ACTIVESEEDING_UNINSTALLED)
- }),
- new JoinClause("t1."+jobs.idField,"t0."+jobQueue.jobIDField)}))
- .append(") ");
+ // This query MUST return only documents that are in a pending state which belong to an active job!!!
- sb.append(database.constructOffsetLimitClause(0,n));
+ sb.append(jobQueue.idField).append(",")
+ .append(jobQueue.docHashField).append(",")
+ .append(jobQueue.docIDField).append(",")
+ .append(jobQueue.jobIDField)
+ .append(" FROM ").append(jobQueue.getTableName()).append(" WHERE ")
+ .append(database.buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(jobQueue.needPriorityField,jobQueue.needPriorityToString(JobQueue.NEEDPRIORITY_TRUE))})).append(" ")
+ .append(database.constructOffsetLimitClause(0,n));
- // Analyze jobqueue tables unconditionally, since it's become much more sensitive in 8.3 than it used to be.
- //jobQueue.unconditionallyAnalyzeTables();
+ // Analyze jobqueue tables unconditionally, since it's become much more sensitive in 8.3 than it used to be.
+ //jobQueue.unconditionallyAnalyzeTables();
- //IResultSet set = database.performQuery(sb.toString(),list,null,null,n,null);
- IResultSet set = database.performQuery(sb.toString(),list,null,null);
+ //IResultSet set = database.performQuery(sb.toString(),list,null,null,n,null);
+ IResultSet set = database.performQuery(sb.toString(),list,null,null);
- DocumentDescription[] rval = new DocumentDescription[set.getRowCount()];
+ DocumentDescription[] rval = new DocumentDescription[set.getRowCount()];
+ // To avoid deadlock, we want to update the document id hashes in order. This means reading into a structure I can sort by docid hash,
+ // before updating any rows in jobqueue.
+ String[] docIDHashes = new String[set.getRowCount()];
+ Map<String,DocumentDescription> storageMap = new HashMap<String,DocumentDescription>();
- int i = 0;
- while (i < set.getRowCount())
- {
- IResultRow row = set.getRow(i);
- rval[i] =new DocumentDescription((Long)row.getValue(jobQueue.idField),
- (Long)row.getValue(jobQueue.jobIDField),
- (String)row.getValue(jobQueue.docHashField),
- (String)row.getValue(jobQueue.docIDField));
- i++;
- }
+ for (int i = 0 ; i < set.getRowCount() ; i++)
+ {
+ IResultRow row = set.getRow(i);
+ String docHash = (String)row.getValue(jobQueue.docHashField);
+ Long jobID = (Long)row.getValue(jobQueue.jobIDField);
+ rval[i] = new DocumentDescription((Long)row.getValue(jobQueue.idField),
+ jobID,
+ docHash,
+ (String)row.getValue(jobQueue.docIDField));
+ // Compute the doc ID hash
+ docIDHashes[i] = docHash + ":" + jobID;
+ storageMap.put(docIDHashes[i],rval[i]);
+ }
+
+ java.util.Arrays.sort(docIDHashes);
+ for (String docIDHash : docIDHashes)
+ {
+ DocumentDescription dd = storageMap.get(docIDHash);
+ jobQueue.markNeedPriorityInProgress(dd.getID(),processID);
+ }
- return rval;
+ database.performCommit();
+ return rval;
+ }
+ catch (ManifoldCFException e)
+ {
+ database.signalRollback();
+ if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
+ {
+ if (Logging.perf.isDebugEnabled())
+ Logging.perf.debug("Aborted transaction finding documents for reprioritization: "+e.getMessage());
+ sleepAmt = getRandomAmount();
+ continue;
+ }
+ throw e;
+ }
+ catch (Error e)
+ {
+ database.signalRollback();
+ throw e;
+ }
+ finally
+ {
+ database.endTransaction();
+ }
+ }
+ finally
+ {
+ lockManager.leaveWriteLock(reprioritizationLock);
+ sleepFor(sleepAmt);
+ }
+ }
}
/** Save a set of document priorities. In the case where a document was eligible to have its
* priority set, but it no longer is eligible, then the provided priority will not be written.
- *@param currentTime is the time in milliseconds since epoch.
*@param documentDescriptions are the document descriptions.
*@param priorities are the desired priorities.
*/
@Override
- public void writeDocumentPriorities(long currentTime, DocumentDescription[] documentDescriptions, IPriorityCalculator[] priorities)
+ public void writeDocumentPriorities(DocumentDescription[] documentDescriptions, IPriorityCalculator[] priorities)
throws ManifoldCFException
{
@@ -2154,16 +2228,14 @@ public class JobManager implements IJobM
while (true)
{
// This should be ordered by document identifier hash in order to prevent potential deadlock conditions
- HashMap indexMap = new HashMap();
+ Map<String,Integer> indexMap = new HashMap<String,Integer>();
String[] docIDHashes = new String[documentDescriptions.length];
- int i = 0;
- while (i < documentDescriptions.length)
+ for (int i = 0; i < documentDescriptions.length; i++)
{
String documentIDHash = documentDescriptions[i].getDocumentIdentifierHash() + ":"+documentDescriptions[i].getJobID();
docIDHashes[i] = documentIDHash;
indexMap.put(documentIDHash,new Integer(i));
- i++;
}
java.util.Arrays.sort(docIDHashes);
@@ -2176,18 +2248,31 @@ public class JobManager implements IJobM
{
// Need to order the writes by doc id.
- i = 0;
- while (i < docIDHashes.length)
+ for (int i = 0; i < docIDHashes.length; i++)
{
String docIDHash = docIDHashes[i];
- Integer x = (Integer)indexMap.remove(docIDHash);
+ Integer x = indexMap.remove(docIDHash);
if (x == null)
throw new ManifoldCFException("Assertion failure: duplicate document identifier jobid/hash detected!");
int index = x.intValue();
DocumentDescription dd = documentDescriptions[index];
- IPriorityCalculator priority = priorities[index];
- jobQueue.writeDocPriority(currentTime,dd.getID(),priority);
- i++;
+ // Query for the status
+ ArrayList list = new ArrayList();
+ String query = database.buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(jobQueue.idField,dd.getID())});
+ IResultSet set = database.performQuery("SELECT "+jobQueue.needPriorityField+" FROM "+jobQueue.getTableName()+" WHERE "+
+ query+" FOR UPDATE",list,null,null);
+ if (set.getRowCount() > 0)
+ {
+ IResultRow row = set.getRow(0);
+ // Grab the needPriority value
+ int needPriority = jobQueue.stringToNeedPriority((String)row.getValue(jobQueue.needPriorityField));
+ if (needPriority == JobQueue.NEEDPRIORITY_INPROGRESS)
+ {
+ IPriorityCalculator priority = priorities[index];
+ jobQueue.writeDocPriority(dd.getID(),priority);
+ }
+ }
}
database.performCommit();
break;
@@ -2517,10 +2602,7 @@ public class JobManager implements IJobM
// But, after I did this, it was no longer necessary to have such a large transaction either.
- // Anything older than 10 minutes ago is considered eligible for reprioritization.
- long prioritizationTime = currentTime - 60000L * 10L;
-
- ThrottleLimit vList = new ThrottleLimit(n,prioritizationTime);
+ ThrottleLimit vList = new ThrottleLimit(n);
IResultSet jobconnections = jobs.getActiveJobConnections();
HashMap connectionSet = new HashMap();
@@ -2802,8 +2884,7 @@ public class JobManager implements IJobM
.append(jobQueue.docIDField).append(",t0.")
.append(jobQueue.statusField).append(",t0.")
.append(jobQueue.failTimeField).append(",t0.")
- .append(jobQueue.failCountField).append(",t0.")
- .append(jobQueue.prioritySetField).append(" FROM ").append(jobQueue.getTableName())
+ .append(jobQueue.failCountField).append(" FROM ").append(jobQueue.getTableName())
.append(" t0 ").append(jobQueue.getGetNextDocumentsIndexHint()).append(" WHERE ");
sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{
@@ -2908,8 +2989,8 @@ public class JobManager implements IJobM
// To avoid deadlock, we want to update the document id hashes in order. This means reading into a structure I can sort by docid hash,
// before updating any rows in jobqueue.
String[] docIDHashes = new String[set.getRowCount()];
- Map storageMap = new HashMap();
- Map statusMap = new HashMap();
+ Map<String,DocumentDescription> storageMap = new HashMap<String,DocumentDescription>();
+ Map<String,Integer> statusMap = new HashMap<String,Integer>();
int i = 0;
while (i < set.getRowCount())
@@ -4343,14 +4424,13 @@ public class JobManager implements IJobM
*@param docIDs are the local document identifiers.
*@param overrideSchedule is true if any existing document schedule should be overridden.
*@param hopcountMethod is either accurate, nodelete, or neverdelete.
- *@param currentTime is the current time in milliseconds since epoch.
*@param documentPriorities are the document priorities corresponding to the document identifiers.
*@param prereqEventNames are the events that must be completed before each document can be processed.
*/
@Override
public void addDocumentsInitial(String processID, Long jobID, String[] legalLinkTypes,
String[] docIDHashes, String[] docIDs, boolean overrideSchedule,
- int hopcountMethod, long currentTime, IPriorityCalculator[] documentPriorities,
+ int hopcountMethod, IPriorityCalculator[] documentPriorities,
String[][] prereqEventNames)
throws ManifoldCFException
{
@@ -4437,12 +4517,12 @@ public class JobManager implements IJobM
int status = jobQueue.stringToStatus((String)row.getValue(jobQueue.statusField));
Long checkTimeValue = (Long)row.getValue(jobQueue.checkTimeField);
- jobQueue.updateExistingRecordInitial(rowID,status,checkTimeValue,executeTime,currentTime,docPriority,docPrereqs,processID);
+ jobQueue.updateExistingRecordInitial(rowID,status,checkTimeValue,executeTime,docPriority,docPrereqs,processID);
}
else
{
// Not found. Attempt an insert instead. This may fail due to constraints, but if this happens, the whole transaction will be retried.
- jobQueue.insertNewRecordInitial(jobID,docIDHash,docID,docPriority,executeTime,currentTime,docPrereqs,processID);
+ jobQueue.insertNewRecordInitial(jobID,docIDHash,docID,docPriority,executeTime,docPrereqs,processID);
}
z++;
@@ -4868,7 +4948,7 @@ public class JobManager implements IJobM
String[] docIDHashes, String[] docIDs,
String parentIdentifierHash, String relationshipType,
int hopcountMethod, String[][] dataNames, Object[][][] dataValues,
- long currentTime, IPriorityCalculator[] documentPriorities,
+ IPriorityCalculator[] documentPriorities,
String[][] prereqEventNames)
throws ManifoldCFException
{
@@ -5043,7 +5123,7 @@ public class JobManager implements IJobM
else
{
// Not found. Attempt an insert instead. This may fail due to constraints, but if this happens, the whole transaction will be retried.
- jobQueue.insertNewRecord(jobID,docIDHash,reorderedDocumentIdentifiers[z],reorderedDocumentPriorities[z],0L,currentTime,reorderedDocumentPrerequisites[z]);
+ jobQueue.insertNewRecord(jobID,docIDHash,reorderedDocumentIdentifiers[z],reorderedDocumentPriorities[z],0L,reorderedDocumentPrerequisites[z]);
}
}
@@ -5069,7 +5149,7 @@ public class JobManager implements IJobM
// helps us determine whether we're going to need to "flip" HOPCOUNTREMOVED documents
// to the PENDING state. If the new link ended in an existing record, THEN we need to flip them all!
jobQueue.updateExistingRecord(jr.getRecordID(),jr.getStatus(),jr.getCheckTimeValue(),
- 0L,currentTime,carrydownChangesSeen[z] || (hopcountChangesSeen!=null && hopcountChangesSeen[z]),
+ 0L,carrydownChangesSeen[z] || (hopcountChangesSeen!=null && hopcountChangesSeen[z]),
reorderedDocumentPriorities[z],reorderedDocumentPrerequisites[z]);
// Signal if we need to perform the flip
if (hopcountChangesSeen != null && hopcountChangesSeen[z])
@@ -5140,7 +5220,6 @@ public class JobManager implements IJobM
*@param hopcountMethod is the desired method for managing hopcounts.
*@param dataNames are the names of the data to carry down to the child from this parent.
*@param dataValues are the values to carry down to the child from this parent, corresponding to dataNames above.
- *@param currentTime is the time in milliseconds since epoch that will be recorded for this operation.
*@param priority is the desired document priority for the document.
*@param prereqEventNames are the events that must be completed before the document can be processed.
*/
@@ -5149,13 +5228,13 @@ public class JobManager implements IJobM
Long jobID, String[] legalLinkTypes, String docIDHash, String docID,
String parentIdentifierHash, String relationshipType,
int hopcountMethod, String[] dataNames, Object[][] dataValues,
- long currentTime, IPriorityCalculator priority, String[] prereqEventNames)
+ IPriorityCalculator priority, String[] prereqEventNames)
throws ManifoldCFException
{
addDocuments(processID,jobID,legalLinkTypes,
new String[]{docIDHash},new String[]{docID},
parentIdentifierHash,relationshipType,hopcountMethod,new String[][]{dataNames},
- new Object[][][]{dataValues},currentTime,new IPriorityCalculator[]{priority},new String[][]{prereqEventNames});
+ new Object[][][]{dataValues},new IPriorityCalculator[]{priority},new String[][]{prereqEventNames});
}
/** Undo the addition of child documents to the queue, for a set of documents.
@@ -5507,7 +5586,7 @@ public class JobManager implements IJobM
*@param docPriorities are the document priorities to assign to the documents, if needed.
*/
@Override
- public void carrydownChangeDocumentMultiple(DocumentDescription[] documentDescriptions, long currentTime, IPriorityCalculator[] docPriorities)
+ public void carrydownChangeDocumentMultiple(DocumentDescription[] documentDescriptions, IPriorityCalculator[] docPriorities)
throws ManifoldCFException
{
if (documentDescriptions.length == 0)
@@ -5587,7 +5666,7 @@ public class JobManager implements IJobM
if (jr != null)
// It was an existing row; do the update logic; use the 'carrydown changes' flag = true all the time.
jobQueue.updateExistingRecord(jr.getRecordID(),jr.getStatus(),jr.getCheckTimeValue(),
- 0L,currentTime,true,docPriorities[originalIndex],null);
+ 0L,true,docPriorities[originalIndex],null);
j++;
}
database.performCommit();
@@ -5625,10 +5704,10 @@ public class JobManager implements IJobM
*@param docPriority is the document priority to assign to the document, if needed.
*/
@Override
- public void carrydownChangeDocument(DocumentDescription documentDescription, long currentTime, IPriorityCalculator docPriority)
+ public void carrydownChangeDocument(DocumentDescription documentDescription, IPriorityCalculator docPriority)
throws ManifoldCFException
{
- carrydownChangeDocumentMultiple(new DocumentDescription[]{documentDescription},currentTime,new IPriorityCalculator[]{docPriority});
+ carrydownChangeDocumentMultiple(new DocumentDescription[]{documentDescription},new IPriorityCalculator[]{docPriority});
}
/** Sleep a random amount of time after a transaction abort.
@@ -8198,6 +8277,7 @@ public class JobManager implements IJobM
// All the job's documents need to have their docpriority set to null, to clear dead wood out of the docpriority index.
// See CONNECTORS-290.
// We do this BEFORE updating the job state.
+
jobQueue.clearDocPriorities(jobID);
IJobDescription jobDesc = jobs.load(jobID,true);
@@ -9495,30 +9575,30 @@ public class JobManager implements IJobM
{
// For each connection, there is (a) a number (which is the maximum per bin), and (b)
// a current running count per bin. These are stored as elements in a hash map.
- protected HashMap connectionMap = new HashMap();
+ protected Map<String,ThrottleJobItem> connectionMap = new HashMap<String,ThrottleJobItem>();
// The maximum number of jobs that have reached their chunk size limit that we
// need
- protected int n;
+ protected final int n;
// This is the hash table that maps a job ID to the object that tracks the number
// of documents already accumulated for this resultset. The count of the number
// of queue records we have is tallied by going through each job in this table
// and adding the records outstanding for it.
- protected HashMap jobQueueHash = new HashMap();
+ protected final Map<Long,QueueHashItem> jobQueueHash = new HashMap<Long,QueueHashItem>();
// This is the map from jobid to connection name
- protected HashMap jobConnection = new HashMap();
+ protected Map<Long,String> jobConnection = new HashMap<Long,String>();
// This is the set of allowed connection names. We discard all documents that are
// not from that set.
- protected HashMap activeConnections = new HashMap();
+ protected Map<String,IRepositoryConnector> activeConnections = new HashMap<String,IRepositoryConnector>();
// This is the number of documents per set per connection.
- protected HashMap setSizes = new HashMap();
+ protected Map<String,Integer> setSizes = new HashMap<String,Integer>();
// These are the individual connection maximums, keyed by connection name.
- protected HashMap maxConnectionCounts = new HashMap();
+ protected Map<String,MutableInteger> maxConnectionCounts = new HashMap<String,MutableInteger>();
// This is the maximum number of documents per set over all the connections we are looking at. This helps us establish a sanity limit.
protected int maxSetSize = 0;
@@ -9527,19 +9607,15 @@ public class JobManager implements IJobM
protected int documentsProcessed = 0;
// This is where we accumulate blocking documents. This is an arraylist of DocumentDescription objects.
- protected ArrayList blockingDocumentArray = new ArrayList();
-
- // Cutoff time for documents eligible for prioritization
- protected long prioritizationTime;
+ protected final List<DocumentDescription> blockingDocumentArray = new ArrayList<DocumentDescription>();
/** Constructor.
* This class is built up piecemeal, so the constructor does nothing.
*@param n is the maximum number of full job descriptions we want at this time.
*/
- public ThrottleLimit(int n, long prioritizationTime)
+ public ThrottleLimit(int n)
{
this.n = n;
- this.prioritizationTime = prioritizationTime;
Logging.perf.debug("Limit instance created");
}
@@ -9588,7 +9664,7 @@ public class JobManager implements IJobM
if (Logging.perf.isDebugEnabled())
Logging.perf.debug(" Adding fetch limit of "+Integer.toString(upperLimit)+" fetches for expression '"+regexp+"' for connection '"+connectionName+"'");
- ThrottleJobItem ji = (ThrottleJobItem)connectionMap.get(connectionName);
+ ThrottleJobItem ji = connectionMap.get(connectionName);
if (ji == null)
{
ji = new ThrottleJobItem();
@@ -9629,7 +9705,7 @@ public class JobManager implements IJobM
/** Make a deep copy */
public ThrottleLimit makeDeepCopy()
{
- ThrottleLimit rval = new ThrottleLimit(n,prioritizationTime);
+ ThrottleLimit rval = new ThrottleLimit(n);
// Create a true copy of all the structures in which counts are kept. The referential structures (e.g. connection hashes)
// do not need a deep copy.
rval.activeConnections = activeConnections;
@@ -9639,18 +9715,13 @@ public class JobManager implements IJobM
rval.jobConnection = jobConnection;
// The structures where counts are maintained DO need a deep copy.
rval.documentsProcessed = documentsProcessed;
- Iterator iter;
- iter = connectionMap.keySet().iterator();
- while (iter.hasNext())
+ for (String key : connectionMap.keySet())
{
- Object key = iter.next();
- rval.connectionMap.put(key,((ThrottleJobItem)connectionMap.get(key)).duplicate());
+ rval.connectionMap.put(key,connectionMap.get(key).duplicate());
}
- iter = jobQueueHash.keySet().iterator();
- while (iter.hasNext())
+ for (Long key : jobQueueHash.keySet())
{
- Object key = iter.next();
- rval.jobQueueHash.put(key,((QueueHashItem)jobQueueHash.get(key)).duplicate());
+ rval.jobQueueHash.put(key,jobQueueHash.get(key).duplicate());
}
return rval;
}
@@ -9698,13 +9769,13 @@ public class JobManager implements IJobM
Long jobIDValue = (Long)row.getValue(JobQueue.jobIDField);
// Get the connection name for this row
- String connectionName = (String)jobConnection.get(jobIDValue);
+ String connectionName = jobConnection.get(jobIDValue);
if (connectionName == null)
{
Logging.perf.debug(" Row does not have an eligible job - excluding");
return false;
}
- IRepositoryConnector connectorInstance = (IRepositoryConnector)activeConnections.get(connectionName);
+ IRepositoryConnector connectorInstance = activeConnections.get(connectionName);
if (connectorInstance == null)
{
Logging.perf.debug(" Row does not have an eligible connector instance - excluding");
@@ -9712,7 +9783,7 @@ public class JobManager implements IJobM
}
// Find the connection limit for this document
- MutableInteger connectionLimit = (MutableInteger)maxConnectionCounts.get(connectionName);
+ MutableInteger connectionLimit = maxConnectionCounts.get(connectionName);
if (connectionLimit != null)
{
if (connectionLimit.intValue() == 0)
@@ -9724,11 +9795,11 @@ public class JobManager implements IJobM
}
// Tally this item in the job queue hash, so we can detect when to stop
- QueueHashItem queueItem = (QueueHashItem)jobQueueHash.get(jobIDValue);
+ QueueHashItem queueItem = jobQueueHash.get(jobIDValue);
if (queueItem == null)
{
// Need to talk to the connector to get a max number of docs per chunk
- int maxCount = ((Integer)setSizes.get(connectionName)).intValue();
+ int maxCount = setSizes.get(connectionName).intValue();
queueItem = new QueueHashItem(maxCount);
jobQueueHash.put(jobIDValue,queueItem);
@@ -9744,7 +9815,7 @@ public class JobManager implements IJobM
documentsProcessed++;
//scanRecord.addBins(binNames);
- ThrottleJobItem item = (ThrottleJobItem)connectionMap.get(connectionName);
+ ThrottleJobItem item = connectionMap.get(connectionName);
// If there is no schedule-based throttling on this connection, we're done.
if (item == null)
@@ -9763,8 +9834,11 @@ public class JobManager implements IJobM
if (Logging.perf.isDebugEnabled())
Logging.perf.debug(" Bin "+binNames[j]+" has no more available fetches - excluding");
- Object o = row.getValue(JobQueue.prioritySetField);
- if (o == null || ((Long)o).longValue() <= prioritizationTime)
+ //???
+ //Object o = row.getValue(JobQueue.prioritySetField);
+ //if (o == null || ((Long)o).longValue() <= prioritizationTime)
+ // Fully enabling blocking document logic, for now.
+ if (true)
{
// Need to add a document descriptor based on this row to the blockingDocuments object!
// This will cause it to be reprioritized preferentially, getting it out of the way if it shouldn't
@@ -9799,12 +9873,10 @@ public class JobManager implements IJobM
return false;
// If the number of chunks exceeds n, we are done
- Iterator iter = jobQueueHash.keySet().iterator();
int count = 0;
- while (iter.hasNext())
+ for (Long jobID : jobQueueHash.keySet())
{
- Long jobID = (Long)iter.next();
- QueueHashItem item = (QueueHashItem)jobQueueHash.get(jobID);
+ QueueHashItem item = jobQueueHash.get(jobID);
count += item.getChunkCount();
if (count > n)
return false;
@@ -9821,7 +9893,7 @@ public class JobManager implements IJobM
protected static class QueueHashItem
{
// The number of items per chunk for this job
- int itemsPerChunk;
+ final int itemsPerChunk;
// The number of chunks so far, INCLUDING incomplete chunks
int chunkCount = 0;
// The number of documents in the current incomplete chunk
@@ -9873,10 +9945,10 @@ public class JobManager implements IJobM
protected static class ThrottleJobItem
{
/** These are the bin limits. This is an array of ThrottleLimitSpec objects. */
- protected ArrayList throttleLimits = new ArrayList();
+ protected List<ThrottleLimitSpec> throttleLimits = new ArrayList<ThrottleLimitSpec>();
/** This is a map of the bins and their current counts. If an entry doesn't exist, it's considered to be
* the same as maxBinCount. */
- protected HashMap binCounts = new HashMap();
+ protected final Map<String,MutableInteger> binCounts = new HashMap<String,MutableInteger>();
/** Constructor. */
public ThrottleJobItem()
@@ -9906,11 +9978,9 @@ public class JobManager implements IJobM
{
ThrottleJobItem rval = new ThrottleJobItem();
rval.throttleLimits = throttleLimits;
- Iterator iter = binCounts.keySet().iterator();
- while (iter.hasNext())
+ for (String key : binCounts.keySet())
{
- String key = (String)iter.next();
- this.binCounts.put(key,((MutableInteger)binCounts.get(key)).duplicate());
+ this.binCounts.put(key,binCounts.get(key).duplicate());
}
return rval;
}
@@ -9921,7 +9991,7 @@ public class JobManager implements IJobM
*/
public boolean isEmpty(String binName)
{
- MutableInteger value = (MutableInteger)binCounts.get(binName);
+ MutableInteger value = binCounts.get(binName);
int remaining;
if (value == null)
{
@@ -9940,7 +10010,7 @@ public class JobManager implements IJobM
*/
public void decrement(String binName)
{
- MutableInteger value = (MutableInteger)binCounts.get(binName);
+ MutableInteger value = binCounts.get(binName);
if (value == null)
{
int x = findMaxCount(binName);
@@ -9985,7 +10055,7 @@ public class JobManager implements IJobM
int i = 0;
while (i < throttleLimits.size())
{
- ThrottleLimitSpec spec = (ThrottleLimitSpec)throttleLimits.get(i++);
+ ThrottleLimitSpec spec = throttleLimits.get(i++);
Pattern p = spec.getRegexp();
Matcher m = p.matcher(binName);
if (m.find())
@@ -10004,9 +10074,9 @@ public class JobManager implements IJobM
protected static class ThrottleLimitSpec
{
/** Regexp */
- protected Pattern regexp;
+ protected final Pattern regexp;
/** The fetch limit for all bins matching that regexp */
- protected int maxCount;
+ protected final int maxCount;
/** Constructor */
public ThrottleLimitSpec(String regexp, int maxCount)
Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java Tue Nov 18 14:30:38 2014
@@ -41,10 +41,11 @@ import java.util.*;
* <tr><td>status</td><td>CHAR(1)</td><td></td></tr>
* <tr><td>isseed</td><td>CHAR(1)</td><td></td></tr>
* <tr><td>docpriority</td><td>FLOAT</td><td></td></tr>
- * <tr><td>priorityset</td><td>BIGINT</td><td></td></tr>
* <tr><td>checkaction</td><td>CHAR(1)</td><td></td></tr>
* <tr><td>processid</td><td>VARCHAR(16)</td><td></td></tr>
* <tr><td>seedingprocessid</td><td>VARCHAR(16)</td><td></td></tr>
+ * <tr><td>needpriority</td><td>CHAR(1)</td><td></td></tr>
+ * <tr><td>needpriorityprocessid</td><td>VARCHAR(16)</td><td></td></tr>
* </table>
* <br><br>
*
@@ -77,6 +78,11 @@ public class JobQueue extends org.apache
public final static int ACTION_RESCAN = 0;
public final static int ACTION_REMOVE = 1;
+ // Need priority status
+ public final static int NEEDPRIORITY_FALSE = 0;
+ public final static int NEEDPRIORITY_INPROGRESS = 1;
+ public final static int NEEDPRIORITY_TRUE = 2;
+
// State descriptions are as follows:
// PENDING means a newly-added reference that has not been scanned before.
// ACTIVE means a newly-added reference that is being scanned for the first time.
@@ -114,19 +120,19 @@ public class JobQueue extends org.apache
public static final String failCountField = "failcount";
public static final String isSeedField = "isseed";
public static final String docPriorityField = "docpriority";
- public static final String prioritySetField = "priorityset";
public static final String checkActionField = "checkaction";
public static final String processIDField = "processid";
public static final String seedingProcessIDField = "seedingprocessid";
+ public static final String needPriorityField = "needpriority";
+ public static final String needPriorityProcessIDField = "needpriorityprocessid";
public static final double noDocPriorityValue = 1e9;
public static final Double nullDocPriority = new Double(noDocPriorityValue + 1.0);
- protected static Map statusMap;
+ protected static final Map<String,Integer> statusMap = new HashMap<String,Integer>();
static
{
- statusMap = new HashMap();
statusMap.put("P",new Integer(STATUS_PENDING));
statusMap.put("A",new Integer(STATUS_ACTIVE));
statusMap.put("C",new Integer(STATUS_COMPLETE));
@@ -142,25 +148,32 @@ public class JobQueue extends org.apache
statusMap.put("H",new Integer(STATUS_HOPCOUNTREMOVED));
}
- protected static Map seedstatusMap;
+ protected static final Map<String,Integer> seedstatusMap = new HashMap<String,Integer>();
static
{
- seedstatusMap = new HashMap();
seedstatusMap.put("F",new Integer(SEEDSTATUS_NOTSEED));
seedstatusMap.put("S",new Integer(SEEDSTATUS_SEED));
seedstatusMap.put("N",new Integer(SEEDSTATUS_NEWSEED));
}
- protected static Map actionMap;
+ protected static final Map<String,Integer> actionMap = new HashMap<String,Integer>();
static
{
- actionMap = new HashMap();
actionMap.put("R",new Integer(ACTION_RESCAN));
actionMap.put("D",new Integer(ACTION_REMOVE));
}
+ protected static final Map<String,Integer> needPriorityMap = new HashMap<String,Integer>();
+
+ static
+ {
+ needPriorityMap.put("T",new Integer(NEEDPRIORITY_TRUE));
+ needPriorityMap.put("I",new Integer(NEEDPRIORITY_INPROGRESS));
+ needPriorityMap.put("F",new Integer(NEEDPRIORITY_FALSE));
+ }
+
/** Prerequisite event manager */
protected PrereqEventManager prereqEventManager;
@@ -205,23 +218,20 @@ public class JobQueue extends org.apache
map.put(statusField,new ColumnDescription("CHAR(1)",false,false,null,null,false));
map.put(isSeedField,new ColumnDescription("CHAR(1)",false,true,null,null,false));
map.put(docPriorityField,new ColumnDescription("FLOAT",false,true,null,null,false));
- map.put(prioritySetField,new ColumnDescription("BIGINT",false,true,null,null,false));
map.put(checkActionField,new ColumnDescription("CHAR(1)",false,true,null,null,false));
map.put(processIDField,new ColumnDescription("VARCHAR(16)",false,true,null,null,false));
map.put(seedingProcessIDField,new ColumnDescription("VARCHAR(16)",false,true,null,null,false));
+ map.put(needPriorityField,new ColumnDescription("CHAR(1)",false,true,null,null,false));
+ map.put(needPriorityProcessIDField,new ColumnDescription("VARCHAR(16)",false,true,null,null,false));
performCreate(map,null);
}
else
{
- // Upgrade; null docpriority fields bashed to 'infinity', so they don't slow down MySQL
+ // Upgrade; null docpriority fields bashed to 'infinity', so they don't slow down the docpriority index
Map map = new HashMap();
map.put(docPriorityField,nullDocPriority);
performUpdate(map,"WHERE "+docPriorityField+" IS NULL",null,null);
- map = new HashMap();
- map.put(prioritySetField,new Long(0L));
- performUpdate(map,"WHERE "+prioritySetField+" IS NULL",null,null);
-
// Also, add processIDField
if (existing.get(processIDField) == null)
{
@@ -237,6 +247,92 @@ public class JobQueue extends org.apache
insertMap.put(seedingProcessIDField,new ColumnDescription("VARCHAR(16)",false,true,null,null,false));
performAlter(insertMap,null,null,null);
}
+
+ if (existing.get(needPriorityField) == null && existing.get(needPriorityProcessIDField) == null)
+ {
+ Map insertMap = new HashMap();
+ insertMap.put(needPriorityField,new ColumnDescription("CHAR(1)",false,true,null,null,false));
+ insertMap.put(needPriorityProcessIDField,new ColumnDescription("VARCHAR(16)",false,true,null,null,false));
+ performAlter(insertMap,null,null,null);
+
+ // Add the index now, because we need it for the upgrade
+ performAddIndex(null,new IndexDescription(false,new String[]{needPriorityField}));
+ }
+
+ // Upgrade: If there's a priorityset field, remove it
+ if (existing.get("priorityset") != null)
+ {
+ List deleteList = new ArrayList();
+ deleteList.add("priorityset");
+ performAlter(null,null,deleteList,null);
+ }
+
+ // Read all jobs and their statuses
+ IResultSet jobStatusResults = performQuery("SELECT "+Jobs.idField+","+Jobs.statusField+" FROM jobs",null,null,null);
+ Map<Long,Integer> statusMap = new HashMap<Long,Integer>();
+ for (int i = 0; i < jobStatusResults.getRowCount(); i++)
+ {
+ IResultRow row = jobStatusResults.getRow(i);
+ statusMap.put((Long)row.getValue(Jobs.idField),new Integer(Jobs.stringToStatus((String)row.getValue(Jobs.statusField))));
+ }
+
+ // The needPriority field should be set only for documents that are active.
+ // And we need to work in small chunks for memory reasons.
+ while (true)
+ {
+ ArrayList params = new ArrayList();
+ IResultSet set = performQuery("SELECT "+idField+","+statusField+","+jobIDField+" FROM "+getTableName()+" WHERE "+
+ buildConjunctionClause(params,new ClauseDescription[]{
+ new NullCheckClause(needPriorityField,true)})+
+ " "+constructOffsetLimitClause(0,10000),
+ params,null,null);
+
+ if (set.getRowCount() == 0)
+ break;
+ for (int i = 0 ; i < set.getRowCount() ; i++)
+ {
+ IResultRow row = set.getRow(i);
+ Long rowID = (Long)row.getValue(idField);
+ int status = stringToStatus((String)row.getValue(statusField));
+ Long jobID = (Long)row.getValue(jobIDField);
+ int jobStatus = statusMap.get(jobID).intValue();
+
+ int needsPriority = NEEDPRIORITY_FALSE;
+ if ((status == STATUS_PENDING ||
+ status == STATUS_ACTIVE ||
+ status == STATUS_PENDINGPURGATORY ||
+ status == STATUS_ACTIVEPURGATORY ||
+ status == STATUS_ACTIVENEEDRESCAN ||
+ status == STATUS_ACTIVENEEDRESCANPURGATORY ||
+ status == STATUS_HOPCOUNTREMOVED) &&
+ (jobStatus == Jobs.STATUS_ACTIVE ||
+ jobStatus == Jobs.STATUS_ACTIVESEEDING ||
+ jobStatus == Jobs.STATUS_ACTIVEWAITING ||
+ jobStatus == Jobs.STATUS_ACTIVEWAITINGSEEDING ||
+ jobStatus == Jobs.STATUS_PAUSING ||
+ jobStatus == Jobs.STATUS_PAUSINGSEEDING ||
+ jobStatus == Jobs.STATUS_PAUSINGWAITING ||
+ jobStatus == Jobs.STATUS_PAUSINGWAITINGSEEDING ||
+ jobStatus == Jobs.STATUS_STARTINGUP ||
+ jobStatus == Jobs.STATUS_STARTINGUPMINIMAL ||
+ jobStatus == Jobs.STATUS_ACTIVE_UNINSTALLED ||
+ jobStatus == Jobs.STATUS_ACTIVESEEDING_UNINSTALLED ||
+ jobStatus == Jobs.STATUS_ACTIVE_NOOUTPUT ||
+ jobStatus == Jobs.STATUS_ACTIVESEEDING_NOOUTPUT ||
+ jobStatus == Jobs.STATUS_ACTIVE_NEITHER ||
+ jobStatus == Jobs.STATUS_ACTIVESEEDING_NEITHER))
+ needsPriority = NEEDPRIORITY_TRUE;
+
+ Map updateMap = new HashMap();
+ updateMap.put(needPriorityField,needPriorityToString(needsPriority));
+ ArrayList updateParams = new ArrayList();
+ updateParams.add(rowID);
+ performUpdate(updateMap,"WHERE "+idField+"=?",updateParams,null);
+
+ }
+ }
+
+
}
// Secondary table installation
@@ -248,7 +344,7 @@ public class JobQueue extends org.apache
IndexDescription jobSeedIndex = new IndexDescription(false,new String[]{isSeedField,jobIDField});
IndexDescription failTimeIndex = new IndexDescription(false,new String[]{failTimeField,jobIDField});
IndexDescription actionTimeStatusIndex = new IndexDescription(false,new String[]{statusField,checkActionField,checkTimeField});
- IndexDescription prioritysetStatusIndex = new IndexDescription(false,new String[]{statusField,prioritySetField});
+ IndexDescription needPriorityIndex = new IndexDescription(false,new String[]{needPriorityField});
// No evidence that the extra fields help at all, for any database...
IndexDescription docpriorityIndex = new IndexDescription(false,new String[]{docPriorityField,statusField,checkActionField,checkTimeField});
@@ -262,6 +358,8 @@ public class JobQueue extends org.apache
if (uniqueIndex != null && id.equals(uniqueIndex))
uniqueIndex = null;
+ else if (needPriorityIndex != null && id.equals(needPriorityIndex))
+ needPriorityIndex = null;
else if (jobStatusIndex != null && id.equals(jobStatusIndex))
jobStatusIndex = null;
else if (jobSeedIndex != null && id.equals(jobSeedIndex))
@@ -270,8 +368,6 @@ public class JobQueue extends org.apache
failTimeIndex = null;
else if (actionTimeStatusIndex != null && id.equals(actionTimeStatusIndex))
actionTimeStatusIndex = null;
- else if (prioritysetStatusIndex != null && id.equals(prioritysetStatusIndex))
- prioritysetStatusIndex = null;
else if (docpriorityIndex != null && id.equals(docpriorityIndex))
docpriorityIndex = null;
else if (indexName.indexOf("_pkey") == -1)
@@ -284,6 +380,9 @@ public class JobQueue extends org.apache
if (jobStatusIndex != null)
performAddIndex(null,jobStatusIndex);
+ if (needPriorityIndex != null)
+ performAddIndex(null,needPriorityIndex);
+
if (jobSeedIndex != null)
performAddIndex(null,jobSeedIndex);
@@ -293,9 +392,6 @@ public class JobQueue extends org.apache
if (actionTimeStatusIndex != null)
performAddIndex(null,actionTimeStatusIndex);
- if (prioritysetStatusIndex != null)
- performAddIndex(null,prioritysetStatusIndex);
-
if (docpriorityIndex != null)
performAddIndex(null,docpriorityIndex);
@@ -366,12 +462,22 @@ public class JobQueue extends org.apache
public void restart(String processID)
throws ManifoldCFException
{
- // Map ACTIVE back to PENDING.
+ // Map NEEDPRIORITY_INPROCESS back to NEEDPRIORITY_TRUE
HashMap map = new HashMap();
- map.put(statusField,statusToString(STATUS_PENDING));
- map.put(processIDField,null);
ArrayList list = new ArrayList();
+ map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_TRUE));
+ map.put(needPriorityProcessIDField,null);
String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(needPriorityField,needPriorityToString(NEEDPRIORITY_INPROGRESS)),
+ new UnitaryClause(processIDField,processID)});
+ performUpdate(map,"WHERE "+query,list,null);
+
+ // Map ACTIVE back to PENDING.
+ map.clear();
+ list.clear();
+ map.put(statusField,statusToString(STATUS_PENDING));
+ map.put(processIDField,null);
+ query = buildConjunctionClause(list,new ClauseDescription[]{
new MultiClause(statusField,new Object[]{
statusToString(STATUS_ACTIVE),
statusToString(STATUS_ACTIVENEEDRESCAN)}),
@@ -423,7 +529,6 @@ public class JobQueue extends org.apache
// values of the reindexing parameters will cause reindexing to occur at this point, but users
// can configure them higher and shut this down.
noteModifications(0,50000,0);
- // Always analyze.
unconditionallyAnalyzeTables();
TrackerClass.noteGlobalChange("Restart");
@@ -434,12 +539,21 @@ public class JobQueue extends org.apache
public void restart()
throws ManifoldCFException
{
- // Map ACTIVE back to PENDING.
+ // Map NEEDPRIORITY_INPROGRESS to NEEDPRIORITY_TRUE
HashMap map = new HashMap();
- map.put(statusField,statusToString(STATUS_PENDING));
- map.put(processIDField,null);
ArrayList list = new ArrayList();
+ map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_TRUE));
+ map.put(needPriorityProcessIDField,null);
String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(needPriorityField,needPriorityToString(NEEDPRIORITY_INPROGRESS))});
+ performUpdate(map,"WHERE "+query,list,null);
+
+ // Map ACTIVE back to PENDING.
+ map.clear();
+ list.clear();
+ map.put(statusField,statusToString(STATUS_PENDING));
+ map.put(processIDField,null);
+ query = buildConjunctionClause(list,new ClauseDescription[]{
new MultiClause(statusField,new Object[]{
statusToString(STATUS_ACTIVE),
statusToString(STATUS_ACTIVENEEDRESCAN)})});
@@ -761,7 +875,7 @@ public class JobQueue extends org.apache
// Map COMPLETE to PENDINGPURGATORY
HashMap map = new HashMap();
map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
- map.put(prioritySetField,new Long(0L));
+ map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_TRUE));
// Do not reset priorities here! They should all be blank at this point.
map.put(checkTimeField,new Long(0L));
map.put(checkActionField,actionToString(ACTION_RESCAN));
@@ -819,7 +933,7 @@ public class JobQueue extends org.apache
// Map COMPLETE to PENDINGPURGATORY.
HashMap map = new HashMap();
map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
- map.put(prioritySetField,new Long(0L));
+ map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_TRUE));
// Do not reset priorities here! They should all be blank at this point.
map.put(checkTimeField,new Long(0L));
map.put(checkActionField,actionToString(ACTION_RESCAN));
@@ -893,12 +1007,27 @@ public class JobQueue extends org.apache
noteModifications(0,0,1);
}
+ /** Prepare to calculate a document priority for a given row. */
+ public void markNeedPriorityInProgress(Long rowID, String processID)
+ throws ManifoldCFException
+ {
+ HashMap map = new HashMap();
+ map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_INPROGRESS));
+ map.put(needPriorityProcessIDField,processID);
+ ArrayList list = new ArrayList();
+ String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(idField,rowID)});
+ performUpdate(map,"WHERE "+query,list,null);
+ noteModifications(0,1,0);
+ }
+
/** Write out a document priority */
- public void writeDocPriority(long currentTime, Long rowID, IPriorityCalculator priority)
+ public void writeDocPriority(Long rowID, IPriorityCalculator priority)
throws ManifoldCFException
{
HashMap map = new HashMap();
- map.put(prioritySetField,new Long(currentTime));
+ map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
+ map.put(needPriorityProcessIDField,null);
map.put(docPriorityField,new Double(priority.getDocumentPriority()));
ArrayList list = new ArrayList();
String query = buildConjunctionClause(list,new ClauseDescription[]{
@@ -907,12 +1036,13 @@ public class JobQueue extends org.apache
noteModifications(0,1,0);
}
- /** Clear all document priorities for a job */
+ /** Clear all document priorities for a job that is going to sleep */
public void clearDocPriorities(Long jobID)
throws ManifoldCFException
{
HashMap map = new HashMap();
- map.put(prioritySetField,new Long(0L));
+ map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
+ map.put(needPriorityProcessIDField,null);
map.put(docPriorityField,nullDocPriority);
ArrayList list = new ArrayList();
String query = buildConjunctionClause(list,new ClauseDescription[]{
@@ -921,6 +1051,22 @@ public class JobQueue extends org.apache
noteModifications(0,1,0);
}
+ /** Clear all document priorities globally for all documents that
+ * have priorities set, and signal that we need new priorities for all.
+ */
+ public void clearAllDocPriorities()
+ throws ManifoldCFException
+ {
+ HashMap map = new HashMap();
+ map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_TRUE));
+ map.put(docPriorityField,nullDocPriority);
+ ArrayList list = new ArrayList();
+ String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(docPriorityField,"<",nullDocPriority)});
+ performUpdate(map,"WHERE "+query,list,null);
+ noteModifications(0,1,0);
+ }
+
/** Set the "completed" status for a record.
*/
public void updateCompletedRecord(Long recID, int currentStatus)
@@ -941,7 +1087,7 @@ public class JobQueue extends org.apache
checkTimeValue = null;
// Remove document priority; we don't want to pollute the queue. See CONNECTORS-290.
map.put(docPriorityField,nullDocPriority);
- map.put(prioritySetField,new Long(0L));
+ map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
break;
case STATUS_ACTIVENEEDRESCAN:
case STATUS_ACTIVENEEDRESCANPURGATORY:
@@ -1214,7 +1360,7 @@ public class JobQueue extends org.apache
* The record is presumed to exist and have been locked, via "FOR UPDATE".
*/
public void updateExistingRecordInitial(Long recordID, int currentStatus, Long checkTimeValue,
- long desiredExecuteTime, long currentTime, IPriorityCalculator desiredPriority, String[] prereqEvents,
+ long desiredExecuteTime, IPriorityCalculator desiredPriority, String[] prereqEvents,
String processID)
throws ManifoldCFException
{
@@ -1253,7 +1399,7 @@ public class JobQueue extends org.apache
map.put(failCountField,null);
// Update the doc priority.
map.put(docPriorityField,new Double(desiredPriority.getDocumentPriority()));
- map.put(prioritySetField,new Long(currentTime));
+ map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
break;
case STATUS_PENDING:
@@ -1310,7 +1456,7 @@ public class JobQueue extends org.apache
*@param docID is the local document identifier.
*/
public void insertNewRecordInitial(Long jobID, String docHash, String docID, IPriorityCalculator desiredDocPriority,
- long desiredExecuteTime, long currentTime, String[] prereqEvents, String processID)
+ long desiredExecuteTime, String[] prereqEvents, String processID)
throws ManifoldCFException
{
// No prerequisites should be possible at this point.
@@ -1330,7 +1476,7 @@ public class JobQueue extends org.apache
map.put(seedingProcessIDField,processID);
// Set the document priority
map.put(docPriorityField,new Double(desiredDocPriority.getDocumentPriority()));
- map.put(prioritySetField,new Long(currentTime));
+ map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
performInsert(map,null);
prereqEventManager.addRows(recordID,prereqEvents);
noteModifications(1,0,0);
@@ -1511,7 +1657,7 @@ public class JobQueue extends org.apache
* The record is presumed to exist and have been locked, via "FOR UPDATE".
*/
public void updateExistingRecord(Long recordID, int currentStatus, Long checkTimeValue,
- long desiredExecuteTime, long currentTime, boolean otherChangesSeen,
+ long desiredExecuteTime, boolean otherChangesSeen,
IPriorityCalculator desiredPriority, String[] prereqEvents)
throws ManifoldCFException
{
@@ -1529,7 +1675,7 @@ public class JobQueue extends org.apache
map.put(failCountField,null);
// Going into pending: set the docpriority.
map.put(docPriorityField,new Double(desiredPriority.getDocumentPriority()));
- map.put(prioritySetField,new Long(currentTime));
+ map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
break;
case STATUS_COMPLETE:
case STATUS_BEINGCLEANED:
@@ -1546,7 +1692,7 @@ public class JobQueue extends org.apache
map.put(failCountField,null);
// Going into pending: set the docpriority.
map.put(docPriorityField,new Double(desiredPriority.getDocumentPriority()));
- map.put(prioritySetField,new Long(currentTime));
+ map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
break;
}
return;
@@ -1575,7 +1721,7 @@ public class JobQueue extends org.apache
map.put(failCountField,null);
// Going into pending: set the docpriority.
map.put(docPriorityField,new Double(desiredPriority.getDocumentPriority()));
- map.put(prioritySetField,new Long(currentTime));
+ map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
break;
}
return;
@@ -1599,7 +1745,7 @@ public class JobQueue extends org.apache
map.put(failCountField,null);
// Going into pending: set the docpriority.
map.put(docPriorityField,new Double(desiredPriority.getDocumentPriority()));
- map.put(prioritySetField,new Long(currentTime));
+ map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
break;
}
return;
@@ -1641,7 +1787,7 @@ public class JobQueue extends org.apache
*
*/
public void insertNewRecord(Long jobID, String docIDHash, String docID, IPriorityCalculator desiredDocPriority, long desiredExecuteTime,
- long currentTime, String[] prereqEvents)
+ String[] prereqEvents)
throws ManifoldCFException
{
HashMap map = new HashMap();
@@ -1655,7 +1801,7 @@ public class JobQueue extends org.apache
map.put(statusField,statusToString(STATUS_PENDING));
// Be sure to set the priority also
map.put(docPriorityField,new Double(desiredDocPriority.getDocumentPriority()));
- map.put(prioritySetField,new Long(currentTime));
+ map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
performInsert(map,null);
prereqEventManager.addRows(recordID,prereqEvents);
noteModifications(1,0,0);
@@ -1690,7 +1836,7 @@ public class JobQueue extends org.apache
{
if (x == null || x.length() == 0)
return SEEDSTATUS_NOTSEED;
- Integer y = (Integer)seedstatusMap.get(x);
+ Integer y = seedstatusMap.get(x);
if (y == null)
throw new ManifoldCFException("Unknown seed status code: "+x);
return y.intValue();
@@ -1701,7 +1847,7 @@ public class JobQueue extends org.apache
public static int stringToAction(String value)
throws ManifoldCFException
{
- Integer x = (Integer)actionMap.get(value);
+ Integer x = actionMap.get(value);
if (x == null)
throw new ManifoldCFException("Unknown action string: '"+value+"'");
return x.intValue();
@@ -1722,6 +1868,37 @@ public class JobQueue extends org.apache
}
}
+ /** Convert need priority value to boolean.
+ */
+ public static int stringToNeedPriority(String value)
+ throws ManifoldCFException
+ {
+ if (value == null || value.length() == 0)
+ return NEEDPRIORITY_FALSE;
+ Integer x = needPriorityMap.get(value);
+ if (x == null)
+ throw new ManifoldCFException("Unknown needpriority string: '"+value+"'");
+ return x.intValue();
+ }
+
+ /** Convert boolean to need priority value.
+ */
+ public static String needPriorityToString(int value)
+ throws ManifoldCFException
+ {
+ switch (value)
+ {
+ case NEEDPRIORITY_TRUE:
+ return "T";
+ case NEEDPRIORITY_FALSE:
+ return "F";
+ case NEEDPRIORITY_INPROGRESS:
+ return "I";
+ default:
+ throw new ManifoldCFException("Bad needpriority value: "+Integer.toString(value));
+ }
+ }
+
/** Convert status field value to integer.
*@param value is the string.
*@return the integer.
@@ -1729,7 +1906,7 @@ public class JobQueue extends org.apache
public static int stringToStatus(String value)
throws ManifoldCFException
{
- Integer x = (Integer)statusMap.get(value);
+ Integer x = statusMap.get(value);
if (x == null)
throw new ManifoldCFException("Unknown status string: '"+value+"'");
return x.intValue();
Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java Tue Nov 18 14:30:38 2014
@@ -39,7 +39,6 @@ public class ReprioritizationTracker imp
protected final static String trackerWriteLock = "_REPR_TRACKER_LOCK_";
protected final static String trackerProcessIDResource = "_REPR_TRACKER_PID_";
protected final static String trackerReproIDResource = "_REPR_TRACKER_RID_";
- protected final static String trackerTimestampResource = "_REPR_TIMESTAMP_";
protected final static String trackerMinimumDepthResource = "_REPR_MINDEPTH_";
/** Lock manager */
@@ -67,21 +66,18 @@ public class ReprioritizationTracker imp
*@param reproID is the reprocessing thread ID
*/
@Override
- public void startReprioritization(long prioritizationTime, String processID, String reproID)
+ public void startReprioritization(String processID, String reproID)
throws ManifoldCFException
{
lockManager.enterWriteLock(trackerWriteLock);
try
{
- Long currentTime = readTime();
String currentProcessID = readProcessID();
- if (currentTime != null && currentProcessID != null)
+ if (currentProcessID != null)
{
// Already a reprioritization in progress.
- if (prioritizationTime <= currentTime.longValue())
- return;
+ return;
}
- writeTime(new Long(prioritizationTime));
writeProcessID(processID);
writeReproID(reproID);
try
@@ -90,7 +86,6 @@ public class ReprioritizationTracker imp
}
catch (Throwable e)
{
- writeTime(null);
writeProcessID(null);
writeReproID(null);
if (e instanceof Error)
@@ -111,30 +106,6 @@ public class ReprioritizationTracker imp
}
- /** Retrieve the current reprioritization time stamp. This should be obtained before
- * performing any prioritization steps.
- *@return the current prioritization timestamp, or null if no prioritization is in effect.
- */
- @Override
- public Long checkReprioritizationInProgress()
- throws ManifoldCFException
- {
- lockManager.enterWriteLock(trackerWriteLock);
- try
- {
- Long currentTime = readTime();
- String currentProcessID = readProcessID();
- String currentReproID = readReproID();
- if (currentTime == null || currentProcessID == null || currentReproID == null)
- return null;
- return currentTime;
- }
- finally
- {
- lockManager.leaveWriteLock(trackerWriteLock);
- }
- }
-
/** Complete a reprioritization activity. Prioritization will be marked as complete
* only if the processID matches the one that started the current reprioritization.
*@param processID is the process ID of the process completing the prioritization.
@@ -146,13 +117,11 @@ public class ReprioritizationTracker imp
lockManager.enterWriteLock(trackerWriteLock);
try
{
- Long currentTime = readTime();
String currentProcessID = readProcessID();
String currentReproID = readReproID();
- if (currentTime != null && currentProcessID != null && currentReproID != null && currentReproID.equals(reproID))
+ if (currentProcessID != null && currentReproID != null && currentReproID.equals(reproID))
{
// Null out the fields
- writeTime(null);
writeProcessID(null);
writeReproID(null);
}
@@ -174,10 +143,9 @@ public class ReprioritizationTracker imp
lockManager.enterWriteLock(trackerWriteLock);
try
{
- Long currentTime = readTime();
String currentProcessID = readProcessID();
String currentReproID = readReproID();
- if (currentTime != null && currentProcessID != null && currentReproID != null && currentProcessID.equals(processID))
+ if (currentProcessID != null && currentReproID != null && currentProcessID.equals(processID))
return currentReproID;
return null;
}
@@ -211,9 +179,8 @@ public class ReprioritizationTracker imp
lockManager.enterWriteLock(trackerWriteLock);
try
{
- Long reproTime = readTime();
String processID = readProcessID();
- if (reproTime == null || processID == null)
+ if (processID == null)
{
double currentMinimumDepth = readMinimumDepth();
@@ -330,55 +297,6 @@ public class ReprioritizationTracker imp
// Protected methods
- /** Read time.
- *@return the time, or null if none.
- */
- protected Long readTime()
- throws ManifoldCFException
- {
- byte[] timeData = lockManager.readData(trackerTimestampResource);
- if (timeData == null || timeData.length != 8)
- {
- return null;
- }
-
- long rval = (((long)timeData[0]) & 0xffL) +
- ((((long)timeData[1]) << 8) & 0xff00L) +
- ((((long)timeData[2]) << 16) & 0xff0000L) +
- ((((long)timeData[3]) << 24) & 0xff000000L) +
- ((((long)timeData[4]) << 32) & 0xff00000000L) +
- ((((long)timeData[5]) << 40) & 0xff0000000000L) +
- ((((long)timeData[6]) << 48) & 0xff000000000000L) +
- ((((long)timeData[7]) << 56) & 0xff00000000000000L);
-
- return new Long(rval);
- }
-
- /** Write time.
- *@param time is the time to write.
- */
- protected void writeTime(Long timeValue)
- throws ManifoldCFException
- {
- if (timeValue == null)
- lockManager.writeData(trackerTimestampResource, null);
- else
- {
- long time = timeValue.longValue();
- byte[] timeData = new byte[8];
- timeData[0] = (byte)(time & 0xffL);
- timeData[1] = (byte)((time >> 8) & 0xffL);
- timeData[2] = (byte)((time >> 16) & 0xffL);
- timeData[3] = (byte)((time >> 24) & 0xffL);
- timeData[4] = (byte)((time >> 32) & 0xffL);
- timeData[5] = (byte)((time >> 40) & 0xffL);
- timeData[6] = (byte)((time >> 48) & 0xffL);
- timeData[7] = (byte)((time >> 56) & 0xffL);
-
- lockManager.writeData(trackerTimestampResource, timeData);
- }
- }
-
/** Read process ID.
*@return processID, or null if none.
*/
Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java Tue Nov 18 14:30:38 2014
@@ -153,7 +153,9 @@ public class CrawlerAgent implements IAg
// to take care of are dangling ones that won't get done because the process that was doing them went
// away. BUT: somebody may have blown away lock info, in which case we won't know anything at all.
// So we do everything in that case.
- ManifoldCF.resetAllDocumentPriorities(threadContext,System.currentTimeMillis(),currentProcessID);
+
+ ManifoldCF.resetAllDocumentPriorities(threadContext,currentProcessID);
+
}
/** Cleanup after agents process.
@@ -170,12 +172,23 @@ public class CrawlerAgent implements IAg
{
IJobManager jobManager = JobManagerFactory.make(threadContext);
jobManager.cleanupProcessData(cleanupProcessID);
+
+ // If one agents process was starting a reprioritization, it could have started the reprioritization sequence, but
+ // failed to complete it. If so, we may need to reset/complete the reprioritization sequence, which is defined as:
+ // - Resetting prioritization parameters
+ // - Removing all existing document priorities
+ // These must go together in order for the reset to be correct.
+
IReprioritizationTracker rt = ReprioritizationTrackerFactory.make(threadContext);
String reproID = rt.isSpecifiedProcessReprioritizing(cleanupProcessID);
if (reproID != null)
{
// We have to take over the prioritization for the process, which apparently died
// in the middle.
+
+ jobManager.clearAllDocumentPriorities();
+
+ /*
IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(threadContext);
// Reprioritize all documents in the jobqueue, 1000 at a time
@@ -188,6 +201,7 @@ public class CrawlerAgent implements IAg
// activity.
// In order for this to be the correct functionality, ALL reseeding and requeuing operations MUST reset the associated document
// priorities.
+ // ??? -- start the process of reprioritization ONLY; don't do the whole thing.
while (true)
{
long startTime = System.currentTimeMillis();
@@ -200,15 +214,16 @@ public class CrawlerAgent implements IAg
}
long updateTime = currentTimeValue.longValue();
- DocumentDescription[] docs = jobManager.getNextNotYetProcessedReprioritizationDocuments(updateTime, 10000);
+ DocumentDescription[] docs = jobManager.getNextNotYetProcessedReprioritizationDocuments(10000);
if (docs.length == 0)
break;
// Calculate new priorities for all these documents
- ManifoldCF.writeDocumentPriorities(threadContext,docs,connectionMap,jobDescriptionMap,updateTime);
+ ManifoldCF.writeDocumentPriorities(threadContext,docs,connectionMap,jobDescriptionMap);
Logging.threads.debug("Reprioritized "+Integer.toString(docs.length)+" not-yet-processed documents in "+new Long(System.currentTimeMillis()-startTime)+" ms");
}
+ */
rt.doneReprioritization(reproID);
}
Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobResetThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobResetThread.java?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobResetThread.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobResetThread.java Tue Nov 18 14:30:38 2014
@@ -106,7 +106,7 @@ public class JobResetThread extends Thre
{
Logging.threads.debug("Job reset thread reprioritizing documents...");
- ManifoldCF.resetAllDocumentPriorities(threadContext,currentTime,processID);
+ ManifoldCF.resetAllDocumentPriorities(threadContext,processID);
Logging.threads.debug("Job reset thread done reprioritizing documents.");
Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java Tue Nov 18 14:30:38 2014
@@ -966,7 +966,7 @@ public class ManifoldCF extends org.apac
}
// Now, requeue the documents with the new priorities
- jobManager.carrydownChangeDocumentMultiple(requeueCandidates,currentTime,docPriorities);
+ jobManager.carrydownChangeDocumentMultiple(requeueCandidates,docPriorities);
}
/** Stuff colons so we can't have conflicts. */
@@ -1013,11 +1013,26 @@ public class ManifoldCF extends org.apac
/** Reset all (active) document priorities. This operation may occur due to various externally-triggered
* events, such a job abort, pause, resume, wait, or unwait.
*/
- public static void resetAllDocumentPriorities(IThreadContext threadContext, long currentTime, String processID)
+ public static void resetAllDocumentPriorities(IThreadContext threadContext, String processID)
throws ManifoldCFException
{
- ILockManager lockManager = LockManagerFactory.make(threadContext);
+ // The reprioritization cycle is as follows now:
+ // (1) We reset the reprioritization tracker, which causes all bins to be be reset, and locks reprioritization so that it is blocked;
+ // (2) We clear all document priorities;
+ // (3) We unlock reprioritization, so that it may proceed.
IJobManager jobManager = JobManagerFactory.make(threadContext);
+ IReprioritizationTracker rt = ReprioritizationTrackerFactory.make(threadContext);
+
+ String reproID = IDFactory.make(threadContext);
+
+ rt.startReprioritization(processID,reproID);
+
+ jobManager.clearAllDocumentPriorities();
+
+ rt.doneReprioritization(reproID);
+
+ /*
+ ILockManager lockManager = LockManagerFactory.make(threadContext);
IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(threadContext);
IReprioritizationTracker rt = ReprioritizationTrackerFactory.make(threadContext);
@@ -1034,6 +1049,7 @@ public class ManifoldCF extends org.apac
// activity.
// In order for this to be the correct functionality, ALL reseeding and requeuing operations MUST reset the associated document
// priorities.
+ // ???? Should only start the process of reprioritization, not complete it.
while (true)
{
long startTime = System.currentTimeMillis();
@@ -1046,24 +1062,24 @@ public class ManifoldCF extends org.apac
}
long updateTime = currentTimeValue.longValue();
- DocumentDescription[] docs = jobManager.getNextNotYetProcessedReprioritizationDocuments(updateTime, 10000);
+ DocumentDescription[] docs = jobManager.getNextNotYetProcessedReprioritizationDocuments(10000);
if (docs.length == 0)
break;
// Calculate new priorities for all these documents
- writeDocumentPriorities(threadContext,docs,connectionMap,jobDescriptionMap,updateTime);
+ writeDocumentPriorities(threadContext,docs,connectionMap,jobDescriptionMap);
Logging.threads.debug("Reprioritized "+Integer.toString(docs.length)+" not-yet-processed documents in "+new Long(System.currentTimeMillis()-startTime)+" ms");
}
rt.doneReprioritization(reproID);
+ */
}
/** Write a set of document priorities, based on the current queue tracker.
*/
public static void writeDocumentPriorities(IThreadContext threadContext, DocumentDescription[] descs,
- Map<String,IRepositoryConnection> connectionMap, Map<Long,IJobDescription> jobDescriptionMap,
- long currentTime)
+ Map<String,IRepositoryConnection> connectionMap, Map<Long,IJobDescription> jobDescriptionMap)
throws ManifoldCFException
{
IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
@@ -1148,7 +1164,7 @@ public class ManifoldCF extends org.apac
rt.preloadBinValues();
// Now, write all the priorities we can.
- jobManager.writeDocumentPriorities(currentTime,descs,priorities);
+ jobManager.writeDocumentPriorities(descs,priorities);
rt.clearPreloadedValues();
}