You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/11/18 15:30:38 UTC

svn commit: r1640350 [1/2] - in /manifoldcf/branches/dev_1x: ./ framework/ framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/ framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ framework/pull-agent/src/main...

Author: kwright
Date: Tue Nov 18 14:30:38 2014
New Revision: 1640350

URL: http://svn.apache.org/r1640350
Log:
Pull up fix for CONNECTORS-1100 from trunk, plus upgrade from the CONNECTORS-1100-1x branch

Modified:
    manifoldcf/branches/dev_1x/   (props changed)
    manifoldcf/branches/dev_1x/CHANGES.txt
    manifoldcf/branches/dev_1x/framework/   (props changed)
    manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java
    manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java
    manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
    manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
    manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java
    manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java
    manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobResetThread.java
    manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
    manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingActivity.java
    manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SetPriorityThread.java
    manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java
    manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java

Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
  Merged /manifoldcf/branches/CONNECTORS-1100:r1637693-1640317
  Merged /manifoldcf/trunk:r1640319

Modified: manifoldcf/branches/dev_1x/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/CHANGES.txt?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/CHANGES.txt (original)
+++ manifoldcf/branches/dev_1x/CHANGES.txt Tue Nov 18 14:30:38 2014
@@ -3,6 +3,10 @@ $Id$
 
 ======================= 1.8-dev =====================
 
+CONNECTORS-1100: Change how reprioritization is done to improve
+overall usability at large scale.
+(Aeham Abushwashi, Karl Wright)
+
 CONNECTORS-1106: Missing state handling for abort case.
 (Kamil Żyta, Karl Wright)
 

Propchange: manifoldcf/branches/dev_1x/framework/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk/framework:r1640319

Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobManager.java Tue Nov 18 14:30:38 2014
@@ -215,23 +215,29 @@ public interface IJobManager
 
   // These methods support the "set doc priority" thread
 
+  /** Clear all document priorities, in preparation for reprioritization of all previously-prioritized documents.
+  * This method is called to start the dynamic reprioritization cycle, which follows this
+  * method with explicit prioritization of all documents, piece-meal, using getNextNotYetProcessedReprioritizationDocuments(),
+  * and writeDocumentPriorities().
+  */
+  public void clearAllDocumentPriorities()
+    throws ManifoldCFException;
+
   /** Get a list of not-yet-processed documents to reprioritize.  Documents in all jobs will be
   * returned by this method.  Up to n document descriptions will be returned.
-  *@param currentTime is the current time stamp for this prioritization pass.  Avoid
-  *  picking up any documents that are labeled with this timestamp or after.
+  *@param processID is the process that requests the reprioritization documents.
   *@param n is the maximum number of document descriptions desired.
   *@return the document descriptions.
   */
-  public DocumentDescription[] getNextNotYetProcessedReprioritizationDocuments(long currentTime, int n)
+  public DocumentDescription[] getNextNotYetProcessedReprioritizationDocuments(String processID, int n)
     throws ManifoldCFException;
 
   /** Save a set of document priorities.  In the case where a document was eligible to have its
   * priority set, but it no longer is eligible, then the provided priority will not be written.
-  *@param currentTime is the time in milliseconds since epoch.
   *@param descriptions are the document descriptions.
   *@param priorities are the desired priorities.
   */
-  public void writeDocumentPriorities(long currentTime, DocumentDescription[] descriptions, IPriorityCalculator[] priorities)
+  public void writeDocumentPriorities(DocumentDescription[] descriptions, IPriorityCalculator[] priorities)
     throws ManifoldCFException;
 
   // This method supports the "expiration" thread
@@ -414,7 +420,7 @@ public interface IJobManager
   *@param documentDescriptions is the set of description objects for the documents that have had their parent carrydown information changed.
   *@param docPriorities are the document priorities to assign to the documents, if needed.
   */
-  public void carrydownChangeDocumentMultiple(DocumentDescription[] documentDescriptions, long currentTime, IPriorityCalculator[] docPriorities)
+  public void carrydownChangeDocumentMultiple(DocumentDescription[] documentDescriptions, IPriorityCalculator[] docPriorities)
     throws ManifoldCFException;
 
   /** Requeue a document because of carrydown changes.
@@ -423,7 +429,7 @@ public interface IJobManager
   *@param documentDescription is the description object for the document that has had its parent carrydown information changed.
   *@param docPriority is the document priority to assign to the document, if needed.
   */
-  public void carrydownChangeDocument(DocumentDescription documentDescription, long currentTime, IPriorityCalculator docPriority)
+  public void carrydownChangeDocument(DocumentDescription documentDescription, IPriorityCalculator docPriority)
     throws ManifoldCFException;
 
   /** Requeue a document for further processing in the future.
@@ -558,14 +564,13 @@ public interface IJobManager
   *@param docIDs are the local document identifiers.
   *@param overrideSchedule is true if any existing document schedule should be overridden.
   *@param hopcountMethod is either accurate, nodelete, or neverdelete.
-  *@param currentTime is the current time in milliseconds since epoch.
   *@param documentPriorities are the document priorities corresponding to the document identifiers.
   *@param prereqEventNames are the events that must be completed before each document can be processed.
   */
   public void addDocumentsInitial(String processID,
     Long jobID, String[] legalLinkTypes,
     String[] docIDHashes, String[] docIDs, boolean overrideSchedule,
-    int hopcountMethod, long currentTime, IPriorityCalculator[] documentPriorities,
+    int hopcountMethod, IPriorityCalculator[] documentPriorities,
     String[][] prereqEventNames)
     throws ManifoldCFException;
 
@@ -650,7 +655,6 @@ public interface IJobManager
   *@param dataNames are the names of the data to carry down to the child from this parent.
   *@param dataValues are the values to carry down to the child from this parent, corresponding to dataNames above.  If CharacterInput objects are passed in here,
   *       it is the caller's responsibility to clean these up.
-  *@param currentTime is the time in milliseconds since epoch that will be recorded for this operation.
   *@param priority is the desired document priority for the document.
   *@param prereqEventNames are the events that must be completed before the document can be processed.
   */
@@ -660,7 +664,7 @@ public interface IJobManager
     String parentIdentifierHash,
     String relationshipType,
     int hopcountMethod, String[] dataNames, Object[][] dataValues,
-    long currentTime, IPriorityCalculator priority, String[] prereqEventNames)
+    IPriorityCalculator priority, String[] prereqEventNames)
     throws ManifoldCFException;
 
   /** Add documents to the queue in bulk.
@@ -680,7 +684,6 @@ public interface IJobManager
   *@param dataNames are the names of the data to carry down to the child from this parent.
   *@param dataValues are the values to carry down to the child from this parent, corresponding to dataNames above.  If CharacterInput objects are passed in here,
   *       it is the caller's responsibility to clean these up.
-  *@param currentTime is the time in milliseconds since epoch that will be recorded for this operation.
   *@param priorities are the desired document priorities for the documents.
   *@param prereqEventNames are the events that must be completed before each document can be processed.
   */
@@ -690,7 +693,7 @@ public interface IJobManager
     String parentIdentifierHash,
     String relationshipType,
     int hopcountMethod, String[][] dataNames, Object[][][] dataValues,
-    long currentTime, IPriorityCalculator[] priorities,
+    IPriorityCalculator[] priorities,
     String[][] prereqEventNames)
     throws ManifoldCFException;
 

Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IReprioritizationTracker.java Tue Nov 18 14:30:38 2014
@@ -31,21 +31,13 @@ public interface IReprioritizationTracke
   public static final String _rcsid = "@(#)$Id$";
 
   /** Start a reprioritization activity.
-  *@param prioritizationTime is the timestamp of the prioritization.
   *@param processID is the process ID of the process performing/waiting for the prioritization
   * to complete.
   *@param reproID is the reprocessing thread ID
   */
-  public void startReprioritization(long prioritizationTime, String processID, String reproID)
+  public void startReprioritization(String processID, String reproID)
     throws ManifoldCFException;
   
-  /** Retrieve the current reprioritization time stamp.  This should be obtained before
-  * performing any prioritization steps.
-  *@return the current prioritization timestamp, or null if no prioritization is in effect.
-  */
-  public Long checkReprioritizationInProgress()
-    throws ManifoldCFException;
-
   /** Complete a reprioritization activity.  Prioritization will be marked as complete
   * only if the processID matches the one that started the current reprioritization.
   *@param processID is the process ID of the process completing the prioritization.
@@ -93,7 +85,7 @@ public interface IReprioritizationTracke
   */
   public void clearPreloadedValues();
 
-  /** Get a bin value.
+  /** Get a bin value.  Must be called INSIDE a transaction.
   *@param binName is the bin name.
   *@param weightedMinimumDepth is the minimum depth to use.
   *@return the bin value.

Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java Tue Nov 18 14:30:38 2014
@@ -33,6 +33,7 @@ public class JobManager implements IJobM
   public static final String _rcsid = "@(#)$Id: JobManager.java 998576 2010-09-19 01:11:02Z kwright $";
 
   protected static final String stufferLock = "_STUFFER_";
+  protected static final String reprioritizationLock = "_REPRIORITIZER_";
   protected static final String deleteStufferLock = "_DELETESTUFFER_";
   protected static final String expireStufferLock = "_EXPIRESTUFFER_";
   protected static final String cleanStufferLock = "_CLEANSTUFFER_";
@@ -2061,92 +2062,165 @@ public class JobManager implements IJobM
 
   // These methods support the reprioritization thread.
 
+  /** Clear all document priorities, in preparation for reprioritization of all previously-prioritized documents.
+  * This method is called to start the dynamic reprioritization cycle, which follows this
+  * method with explicit prioritization of all documents, piece-meal, using getNextNotYetProcessedReprioritizationDocuments(),
+  * and writeDocumentPriorities().
+  */
+  public void clearAllDocumentPriorities()
+    throws ManifoldCFException
+  {
+    // Note: This can be called at any time, even while worker and seeding threads are also reprioritizing documents.
+    
+    // Retry loop - in case we get a deadlock despite our best efforts
+    while (true)
+    {
+      long sleepAmt = 0L;
+
+      // Start the transaction now
+      database.beginTransaction();
+      try
+      {
+        jobQueue.clearAllDocPriorities();
+        database.performCommit();
+        break;
+      }
+      catch (ManifoldCFException e)
+      {
+        database.signalRollback();
+        if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
+        {
+          if (Logging.perf.isDebugEnabled())
+            Logging.perf.debug("Aborted transaction clearing doc priorities for reprioritization: "+e.getMessage());
+          sleepAmt = getRandomAmount();
+          continue;
+        }
+        throw e;
+      }
+      catch (Error e)
+      {
+        database.signalRollback();
+        throw e;
+      }
+      finally
+      {
+        database.endTransaction();
+        sleepFor(sleepAmt);
+      }
+    }
+  }
+
   /** Get a list of not-yet-processed documents to reprioritize.  Documents in all jobs will be
   * returned by this method.  Up to n document descriptions will be returned.
-  *@param currentTime is the current time stamp for this prioritization pass.  Avoid
-  *  picking up any documents that are labeled with this timestamp or after.
+  *@param processID is the process that requests the reprioritization documents.
   *@param n is the maximum number of document descriptions desired.
   *@return the document descriptions.
   */
   @Override
-  public DocumentDescription[] getNextNotYetProcessedReprioritizationDocuments(long currentTime, int n)
+  public DocumentDescription[] getNextNotYetProcessedReprioritizationDocuments(String processID, int n)
     throws ManifoldCFException
   {
-    StringBuilder sb = new StringBuilder("SELECT ");
-    ArrayList list = new ArrayList();
+    // Retry loop - in case we get a deadlock despite our best efforts
+    while (true)
+    {
+      long sleepAmt = 0L;
 
-    // This query MUST return only documents that are in a pending state which belong to an active job!!!
+      // Write lock insures that only one thread cluster-wide can be doing this at a given time, so FOR UPDATE is unneeded.
+      lockManager.enterWriteLock(reprioritizationLock);
+      try
+      {
+        // Start the transaction now
+        database.beginTransaction();
+        try
+        {
 
-    sb.append(jobQueue.idField).append(",")
-      .append(jobQueue.docHashField).append(",")
-      .append(jobQueue.docIDField).append(",")
-      .append(jobQueue.jobIDField)
-      .append(" FROM ").append(jobQueue.getTableName()).append(" t0 WHERE ")
-      .append(database.buildConjunctionClause(list,new ClauseDescription[]{
-        new MultiClause(jobQueue.statusField,new Object[]{
-          JobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVED),
-          JobQueue.statusToString(jobQueue.STATUS_PENDING),
-          JobQueue.statusToString(jobQueue.STATUS_PENDINGPURGATORY)}),
-        new UnitaryClause(jobQueue.prioritySetField,"<",new Long(currentTime))}));
-    
-    sb.append(" AND ")
-      .append(jobQueue.checkActionField).append("=?");
-    list.add(jobQueue.actionToString(JobQueue.ACTION_RESCAN));
+          StringBuilder sb = new StringBuilder("SELECT ");
+          ArrayList list = new ArrayList();
 
-    // Per CONNECTORS-290, we need to be leaving priorities blank for jobs that aren't using them,
-    // so this will be changed to not include jobs where the priorities have been bashed to null.
-    //
-    // I've included ALL states that might have non-null doc priorities.  This includes states
-    // corresponding to uninstalled connectors, since there is no transition that cleans out the
-    // document priorities in these states.  The time during which a connector is uninstalled is
-    // expected to be short, because typically this state is the result of an installation procedure
-    // rather than willful action on the part of a user.
-        
-    sb.append(" AND EXISTS(SELECT 'x' FROM ").append(jobs.getTableName()).append(" t1 WHERE ")
-      .append(database.buildConjunctionClause(list,new ClauseDescription[]{
-        new MultiClause("t1."+jobs.statusField,new Object[]{
-          Jobs.statusToString(Jobs.STATUS_STARTINGUP),
-          Jobs.statusToString(Jobs.STATUS_STARTINGUPMINIMAL),
-          Jobs.statusToString(Jobs.STATUS_ACTIVE),
-          Jobs.statusToString(Jobs.STATUS_ACTIVESEEDING),
-          Jobs.statusToString(Jobs.STATUS_ACTIVE_UNINSTALLED),
-          Jobs.statusToString(Jobs.STATUS_ACTIVESEEDING_UNINSTALLED)
-          }),
-        new JoinClause("t1."+jobs.idField,"t0."+jobQueue.jobIDField)}))
-      .append(") ");
+          // This query MUST return only documents that are in a pending state which belong to an active job!!!
 
-    sb.append(database.constructOffsetLimitClause(0,n));
+          sb.append(jobQueue.idField).append(",")
+            .append(jobQueue.docHashField).append(",")
+            .append(jobQueue.docIDField).append(",")
+            .append(jobQueue.jobIDField)
+            .append(" FROM ").append(jobQueue.getTableName()).append(" WHERE ")
+            .append(database.buildConjunctionClause(list,new ClauseDescription[]{
+              new UnitaryClause(jobQueue.needPriorityField,jobQueue.needPriorityToString(JobQueue.NEEDPRIORITY_TRUE))})).append(" ")
+            .append(database.constructOffsetLimitClause(0,n));
 
-    // Analyze jobqueue tables unconditionally, since it's become much more sensitive in 8.3 than it used to be.
-    //jobQueue.unconditionallyAnalyzeTables();
+          // Analyze jobqueue tables unconditionally, since it's become much more sensitive in 8.3 than it used to be.
+          //jobQueue.unconditionallyAnalyzeTables();
 
-    //IResultSet set = database.performQuery(sb.toString(),list,null,null,n,null);
-    IResultSet set = database.performQuery(sb.toString(),list,null,null);
+          //IResultSet set = database.performQuery(sb.toString(),list,null,null,n,null);
+          IResultSet set = database.performQuery(sb.toString(),list,null,null);
 
-    DocumentDescription[] rval = new DocumentDescription[set.getRowCount()];
+          DocumentDescription[] rval = new DocumentDescription[set.getRowCount()];
+          // To avoid deadlock, we want to update the document id hashes in order.  This means reading into a structure I can sort by docid hash,
+          // before updating any rows in jobqueue.
+          String[] docIDHashes = new String[set.getRowCount()];
+          Map<String,DocumentDescription> storageMap = new HashMap<String,DocumentDescription>();
 
-    int i = 0;
-    while (i < set.getRowCount())
-    {
-      IResultRow row = set.getRow(i);
-      rval[i] =new DocumentDescription((Long)row.getValue(jobQueue.idField),
-        (Long)row.getValue(jobQueue.jobIDField),
-        (String)row.getValue(jobQueue.docHashField),
-        (String)row.getValue(jobQueue.docIDField));
-      i++;
-    }
+          for (int i = 0 ; i < set.getRowCount() ; i++)
+          {
+            IResultRow row = set.getRow(i);
+            String docHash = (String)row.getValue(jobQueue.docHashField);
+            Long jobID = (Long)row.getValue(jobQueue.jobIDField);
+            rval[i] = new DocumentDescription((Long)row.getValue(jobQueue.idField),
+              jobID,
+              docHash,
+              (String)row.getValue(jobQueue.docIDField));
+            // Compute the doc ID hash
+            docIDHashes[i] = docHash + ":" + jobID;
+            storageMap.put(docIDHashes[i],rval[i]);
+          }
+          
+          java.util.Arrays.sort(docIDHashes);
+          for (String docIDHash : docIDHashes)
+          {
+            DocumentDescription dd = storageMap.get(docIDHash);
+            jobQueue.markNeedPriorityInProgress(dd.getID(),processID);
+          }
 
-    return rval;
+          database.performCommit();
+          return rval;
+        }
+        catch (ManifoldCFException e)
+        {
+          database.signalRollback();
+          if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
+          {
+            if (Logging.perf.isDebugEnabled())
+              Logging.perf.debug("Aborted transaction finding documents for reprioritization: "+e.getMessage());
+            sleepAmt = getRandomAmount();
+            continue;
+          }
+          throw e;
+        }
+        catch (Error e)
+        {
+          database.signalRollback();
+          throw e;
+        }
+        finally
+        {
+          database.endTransaction();
+        }
+      }
+      finally
+      {
+        lockManager.leaveWriteLock(reprioritizationLock);
+        sleepFor(sleepAmt);
+      }
+    }
   }
 
   /** Save a set of document priorities.  In the case where a document was eligible to have its
   * priority set, but it no longer is eligible, then the provided priority will not be written.
-  *@param currentTime is the time in milliseconds since epoch.
   *@param documentDescriptions are the document descriptions.
   *@param priorities are the desired priorities.
   */
   @Override
-  public void writeDocumentPriorities(long currentTime, DocumentDescription[] documentDescriptions, IPriorityCalculator[] priorities)
+  public void writeDocumentPriorities(DocumentDescription[] documentDescriptions, IPriorityCalculator[] priorities)
     throws ManifoldCFException
   {
 
@@ -2154,16 +2228,14 @@ public class JobManager implements IJobM
     while (true)
     {
       // This should be ordered by document identifier hash in order to prevent potential deadlock conditions
-      HashMap indexMap = new HashMap();
+      Map<String,Integer> indexMap = new HashMap<String,Integer>();
       String[] docIDHashes = new String[documentDescriptions.length];
 
-      int i = 0;
-      while (i < documentDescriptions.length)
+      for (int i = 0; i < documentDescriptions.length; i++)
       {
         String documentIDHash = documentDescriptions[i].getDocumentIdentifierHash() + ":"+documentDescriptions[i].getJobID();
         docIDHashes[i] = documentIDHash;
         indexMap.put(documentIDHash,new Integer(i));
-        i++;
       }
 
       java.util.Arrays.sort(docIDHashes);
@@ -2176,18 +2248,31 @@ public class JobManager implements IJobM
       {
 
         // Need to order the writes by doc id.
-        i = 0;
-        while (i < docIDHashes.length)
+        for (int i = 0; i < docIDHashes.length; i++)
         {
           String docIDHash = docIDHashes[i];
-          Integer x = (Integer)indexMap.remove(docIDHash);
+          Integer x = indexMap.remove(docIDHash);
           if (x == null)
             throw new ManifoldCFException("Assertion failure: duplicate document identifier jobid/hash detected!");
           int index = x.intValue();
           DocumentDescription dd = documentDescriptions[index];
-          IPriorityCalculator priority = priorities[index];
-          jobQueue.writeDocPriority(currentTime,dd.getID(),priority);
-          i++;
+          // Query for the status
+          ArrayList list = new ArrayList();
+          String query = database.buildConjunctionClause(list,new ClauseDescription[]{
+            new UnitaryClause(jobQueue.idField,dd.getID())});
+          IResultSet set = database.performQuery("SELECT "+jobQueue.needPriorityField+" FROM "+jobQueue.getTableName()+" WHERE "+
+            query+" FOR UPDATE",list,null,null);
+          if (set.getRowCount() > 0)
+          {
+            IResultRow row = set.getRow(0);
+            // Grab the needPriority value
+            int needPriority = jobQueue.stringToNeedPriority((String)row.getValue(jobQueue.needPriorityField));
+            if (needPriority == JobQueue.NEEDPRIORITY_INPROGRESS)
+            {
+              IPriorityCalculator priority = priorities[index];
+              jobQueue.writeDocPriority(dd.getID(),priority);
+            }
+          }
         }
         database.performCommit();
         break;
@@ -2517,10 +2602,7 @@ public class JobManager implements IJobM
     // But, after I did this, it was no longer necessary to have such a large transaction either.
 
 
-    // Anything older than 10 minutes ago is considered eligible for reprioritization.
-    long prioritizationTime = currentTime - 60000L * 10L;
-
-    ThrottleLimit vList = new ThrottleLimit(n,prioritizationTime);
+    ThrottleLimit vList = new ThrottleLimit(n);
 
     IResultSet jobconnections = jobs.getActiveJobConnections();
     HashMap connectionSet = new HashMap();
@@ -2802,8 +2884,7 @@ public class JobManager implements IJobM
       .append(jobQueue.docIDField).append(",t0.")
       .append(jobQueue.statusField).append(",t0.")
       .append(jobQueue.failTimeField).append(",t0.")
-      .append(jobQueue.failCountField).append(",t0.")
-      .append(jobQueue.prioritySetField).append(" FROM ").append(jobQueue.getTableName())
+      .append(jobQueue.failCountField).append(" FROM ").append(jobQueue.getTableName())
       .append(" t0 ").append(jobQueue.getGetNextDocumentsIndexHint()).append(" WHERE ");
     
     sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{
@@ -2908,8 +2989,8 @@ public class JobManager implements IJobM
             // To avoid deadlock, we want to update the document id hashes in order.  This means reading into a structure I can sort by docid hash,
             // before updating any rows in jobqueue.
             String[] docIDHashes = new String[set.getRowCount()];
-            Map storageMap = new HashMap();
-            Map statusMap = new HashMap();
+            Map<String,DocumentDescription> storageMap = new HashMap<String,DocumentDescription>();
+            Map<String,Integer> statusMap = new HashMap<String,Integer>();
 
             int i = 0;
             while (i < set.getRowCount())
@@ -4343,14 +4424,13 @@ public class JobManager implements IJobM
   *@param docIDs are the local document identifiers.
   *@param overrideSchedule is true if any existing document schedule should be overridden.
   *@param hopcountMethod is either accurate, nodelete, or neverdelete.
-  *@param currentTime is the current time in milliseconds since epoch.
   *@param documentPriorities are the document priorities corresponding to the document identifiers.
   *@param prereqEventNames are the events that must be completed before each document can be processed.
   */
   @Override
   public void addDocumentsInitial(String processID, Long jobID, String[] legalLinkTypes,
     String[] docIDHashes, String[] docIDs, boolean overrideSchedule,
-    int hopcountMethod, long currentTime, IPriorityCalculator[] documentPriorities,
+    int hopcountMethod, IPriorityCalculator[] documentPriorities,
     String[][] prereqEventNames)
     throws ManifoldCFException
   {
@@ -4437,12 +4517,12 @@ public class JobManager implements IJobM
             int status = jobQueue.stringToStatus((String)row.getValue(jobQueue.statusField));
             Long checkTimeValue = (Long)row.getValue(jobQueue.checkTimeField);
 
-            jobQueue.updateExistingRecordInitial(rowID,status,checkTimeValue,executeTime,currentTime,docPriority,docPrereqs,processID);
+            jobQueue.updateExistingRecordInitial(rowID,status,checkTimeValue,executeTime,docPriority,docPrereqs,processID);
           }
           else
           {
             // Not found.  Attempt an insert instead.  This may fail due to constraints, but if this happens, the whole transaction will be retried.
-            jobQueue.insertNewRecordInitial(jobID,docIDHash,docID,docPriority,executeTime,currentTime,docPrereqs,processID);
+            jobQueue.insertNewRecordInitial(jobID,docIDHash,docID,docPriority,executeTime,docPrereqs,processID);
           }
 
           z++;
@@ -4868,7 +4948,7 @@ public class JobManager implements IJobM
     String[] docIDHashes, String[] docIDs,
     String parentIdentifierHash, String relationshipType,
     int hopcountMethod, String[][] dataNames, Object[][][] dataValues,
-    long currentTime, IPriorityCalculator[] documentPriorities,
+    IPriorityCalculator[] documentPriorities,
     String[][] prereqEventNames)
     throws ManifoldCFException
   {
@@ -5043,7 +5123,7 @@ public class JobManager implements IJobM
           else
           {
             // Not found.  Attempt an insert instead.  This may fail due to constraints, but if this happens, the whole transaction will be retried.
-            jobQueue.insertNewRecord(jobID,docIDHash,reorderedDocumentIdentifiers[z],reorderedDocumentPriorities[z],0L,currentTime,reorderedDocumentPrerequisites[z]);
+            jobQueue.insertNewRecord(jobID,docIDHash,reorderedDocumentIdentifiers[z],reorderedDocumentPriorities[z],0L,reorderedDocumentPrerequisites[z]);
           }
 
         }
@@ -5069,7 +5149,7 @@ public class JobManager implements IJobM
             // helps us determine whether we're going to need to "flip" HOPCOUNTREMOVED documents
             // to the PENDING state.  If the new link ended in an existing record, THEN we need to flip them all!
             jobQueue.updateExistingRecord(jr.getRecordID(),jr.getStatus(),jr.getCheckTimeValue(),
-              0L,currentTime,carrydownChangesSeen[z] || (hopcountChangesSeen!=null && hopcountChangesSeen[z]),
+              0L,carrydownChangesSeen[z] || (hopcountChangesSeen!=null && hopcountChangesSeen[z]),
               reorderedDocumentPriorities[z],reorderedDocumentPrerequisites[z]);
             // Signal if we need to perform the flip
             if (hopcountChangesSeen != null && hopcountChangesSeen[z])
@@ -5140,7 +5220,6 @@ public class JobManager implements IJobM
   *@param hopcountMethod is the desired method for managing hopcounts.
   *@param dataNames are the names of the data to carry down to the child from this parent.
   *@param dataValues are the values to carry down to the child from this parent, corresponding to dataNames above.
-  *@param currentTime is the time in milliseconds since epoch that will be recorded for this operation.
   *@param priority is the desired document priority for the document.
   *@param prereqEventNames are the events that must be completed before the document can be processed.
   */
@@ -5149,13 +5228,13 @@ public class JobManager implements IJobM
     Long jobID, String[] legalLinkTypes, String docIDHash, String docID,
     String parentIdentifierHash, String relationshipType,
     int hopcountMethod, String[] dataNames, Object[][] dataValues,
-    long currentTime, IPriorityCalculator priority, String[] prereqEventNames)
+    IPriorityCalculator priority, String[] prereqEventNames)
     throws ManifoldCFException
   {
     addDocuments(processID,jobID,legalLinkTypes,
       new String[]{docIDHash},new String[]{docID},
       parentIdentifierHash,relationshipType,hopcountMethod,new String[][]{dataNames},
-      new Object[][][]{dataValues},currentTime,new IPriorityCalculator[]{priority},new String[][]{prereqEventNames});
+      new Object[][][]{dataValues},new IPriorityCalculator[]{priority},new String[][]{prereqEventNames});
   }
 
   /** Undo the addition of child documents to the queue, for a set of documents.
@@ -5507,7 +5586,7 @@ public class JobManager implements IJobM
   *@param docPriorities are the document priorities to assign to the documents, if needed.
   */
   @Override
-  public void carrydownChangeDocumentMultiple(DocumentDescription[] documentDescriptions, long currentTime, IPriorityCalculator[] docPriorities)
+  public void carrydownChangeDocumentMultiple(DocumentDescription[] documentDescriptions, IPriorityCalculator[] docPriorities)
     throws ManifoldCFException
   {
     if (documentDescriptions.length == 0)
@@ -5587,7 +5666,7 @@ public class JobManager implements IJobM
           if (jr != null)
             // It was an existing row; do the update logic; use the 'carrydown changes' flag = true all the time.
             jobQueue.updateExistingRecord(jr.getRecordID(),jr.getStatus(),jr.getCheckTimeValue(),
-              0L,currentTime,true,docPriorities[originalIndex],null);
+              0L,true,docPriorities[originalIndex],null);
           j++;
         }
         database.performCommit();
@@ -5625,10 +5704,10 @@ public class JobManager implements IJobM
   *@param docPriority is the document priority to assign to the document, if needed.
   */
   @Override
-  public void carrydownChangeDocument(DocumentDescription documentDescription, long currentTime, IPriorityCalculator docPriority)
+  public void carrydownChangeDocument(DocumentDescription documentDescription, IPriorityCalculator docPriority)
     throws ManifoldCFException
   {
-    carrydownChangeDocumentMultiple(new DocumentDescription[]{documentDescription},currentTime,new IPriorityCalculator[]{docPriority});
+    carrydownChangeDocumentMultiple(new DocumentDescription[]{documentDescription},new IPriorityCalculator[]{docPriority});
   }
 
   /** Sleep a random amount of time after a transaction abort.
@@ -8198,6 +8277,7 @@ public class JobManager implements IJobM
         // All the job's documents need to have their docpriority set to null, to clear dead wood out of the docpriority index.
         // See CONNECTORS-290.
         // We do this BEFORE updating the job state.
+        
         jobQueue.clearDocPriorities(jobID);
             
         IJobDescription jobDesc = jobs.load(jobID,true);
@@ -9495,30 +9575,30 @@ public class JobManager implements IJobM
   {
     // For each connection, there is (a) a number (which is the maximum per bin), and (b)
     // a current running count per bin.  These are stored as elements in a hash map.
-    protected HashMap connectionMap = new HashMap();
+    protected Map<String,ThrottleJobItem> connectionMap = new HashMap<String,ThrottleJobItem>();
 
     // The maximum number of jobs that have reached their chunk size limit that we
     // need
-    protected int n;
+    protected final int n;
 
     // This is the hash table that maps a job ID to the object that tracks the number
     // of documents already accumulated for this resultset.  The count of the number
     // of queue records we have is tallied by going through each job in this table
     // and adding the records outstanding for it.
-    protected HashMap jobQueueHash = new HashMap();
+    protected final Map<Long,QueueHashItem> jobQueueHash = new HashMap<Long,QueueHashItem>();
 
     // This is the map from jobid to connection name
-    protected HashMap jobConnection = new HashMap();
+    protected Map<Long,String> jobConnection = new HashMap<Long,String>();
 
     // This is the set of allowed connection names.  We discard all documents that are
     // not from that set.
-    protected HashMap activeConnections = new HashMap();
+    protected Map<String,IRepositoryConnector> activeConnections = new HashMap<String,IRepositoryConnector>();
 
     // This is the number of documents per set per connection.
-    protected HashMap setSizes = new HashMap();
+    protected Map<String,Integer> setSizes = new HashMap<String,Integer>();
 
     // These are the individual connection maximums, keyed by connection name.
-    protected HashMap maxConnectionCounts = new HashMap();
+    protected Map<String,MutableInteger> maxConnectionCounts = new HashMap<String,MutableInteger>();
 
     // This is the maximum number of documents per set over all the connections we are looking at.  This helps us establish a sanity limit.
     protected int maxSetSize = 0;
@@ -9527,19 +9607,15 @@ public class JobManager implements IJobM
     protected int documentsProcessed = 0;
 
     // This is where we accumulate blocking documents.  This is an arraylist of DocumentDescription objects.
-    protected ArrayList blockingDocumentArray = new ArrayList();
-
-    // Cutoff time for documents eligible for prioritization
-    protected long prioritizationTime;
+    protected final List<DocumentDescription> blockingDocumentArray = new ArrayList<DocumentDescription>();
 
     /** Constructor.
     * This class is built up piecemeal, so the constructor does nothing.
     *@param n is the maximum number of full job descriptions we want at this time.
     */
-    public ThrottleLimit(int n, long prioritizationTime)
+    public ThrottleLimit(int n)
     {
       this.n = n;
-      this.prioritizationTime = prioritizationTime;
       Logging.perf.debug("Limit instance created");
     }
 
@@ -9588,7 +9664,7 @@ public class JobManager implements IJobM
       if (Logging.perf.isDebugEnabled())
         Logging.perf.debug(" Adding fetch limit of "+Integer.toString(upperLimit)+" fetches for expression '"+regexp+"' for connection '"+connectionName+"'");
 
-      ThrottleJobItem ji = (ThrottleJobItem)connectionMap.get(connectionName);
+      ThrottleJobItem ji = connectionMap.get(connectionName);
       if (ji == null)
       {
         ji = new ThrottleJobItem();
@@ -9629,7 +9705,7 @@ public class JobManager implements IJobM
     /** Make a deep copy */
     public ThrottleLimit makeDeepCopy()
     {
-      ThrottleLimit rval = new ThrottleLimit(n,prioritizationTime);
+      ThrottleLimit rval = new ThrottleLimit(n);
       // Create a true copy of all the structures in which counts are kept.  The referential structures (e.g. connection hashes)
       // do not need a deep copy.
       rval.activeConnections = activeConnections;
@@ -9639,18 +9715,13 @@ public class JobManager implements IJobM
       rval.jobConnection = jobConnection;
       // The structures where counts are maintained DO need a deep copy.
       rval.documentsProcessed = documentsProcessed;
-      Iterator iter;
-      iter = connectionMap.keySet().iterator();
-      while (iter.hasNext())
+      for (String key : connectionMap.keySet())
       {
-        Object key = iter.next();
-        rval.connectionMap.put(key,((ThrottleJobItem)connectionMap.get(key)).duplicate());
+        rval.connectionMap.put(key,connectionMap.get(key).duplicate());
       }
-      iter = jobQueueHash.keySet().iterator();
-      while (iter.hasNext())
+      for (Long key : jobQueueHash.keySet())
       {
-        Object key = iter.next();
-        rval.jobQueueHash.put(key,((QueueHashItem)jobQueueHash.get(key)).duplicate());
+        rval.jobQueueHash.put(key,jobQueueHash.get(key).duplicate());
       }
       return rval;
     }
@@ -9698,13 +9769,13 @@ public class JobManager implements IJobM
       Long jobIDValue = (Long)row.getValue(JobQueue.jobIDField);
 
       // Get the connection name for this row
-      String connectionName = (String)jobConnection.get(jobIDValue);
+      String connectionName = jobConnection.get(jobIDValue);
       if (connectionName == null)
       {
         Logging.perf.debug(" Row does not have an eligible job - excluding");
         return false;
       }
-      IRepositoryConnector connectorInstance = (IRepositoryConnector)activeConnections.get(connectionName);
+      IRepositoryConnector connectorInstance = activeConnections.get(connectionName);
       if (connectorInstance == null)
       {
         Logging.perf.debug(" Row does not have an eligible connector instance - excluding");
@@ -9712,7 +9783,7 @@ public class JobManager implements IJobM
       }
 
       // Find the connection limit for this document
-      MutableInteger connectionLimit = (MutableInteger)maxConnectionCounts.get(connectionName);
+      MutableInteger connectionLimit = maxConnectionCounts.get(connectionName);
       if (connectionLimit != null)
       {
         if (connectionLimit.intValue() == 0)
@@ -9724,11 +9795,11 @@ public class JobManager implements IJobM
       }
 
       // Tally this item in the job queue hash, so we can detect when to stop
-      QueueHashItem queueItem = (QueueHashItem)jobQueueHash.get(jobIDValue);
+      QueueHashItem queueItem = jobQueueHash.get(jobIDValue);
       if (queueItem == null)
       {
         // Need to talk to the connector to get a max number of docs per chunk
-        int maxCount = ((Integer)setSizes.get(connectionName)).intValue();
+        int maxCount = setSizes.get(connectionName).intValue();
         queueItem = new QueueHashItem(maxCount);
         jobQueueHash.put(jobIDValue,queueItem);
 
@@ -9744,7 +9815,7 @@ public class JobManager implements IJobM
       documentsProcessed++;
       //scanRecord.addBins(binNames);
 
-      ThrottleJobItem item = (ThrottleJobItem)connectionMap.get(connectionName);
+      ThrottleJobItem item = connectionMap.get(connectionName);
 
       // If there is no schedule-based throttling on this connection, we're done.
       if (item == null)
@@ -9763,8 +9834,11 @@ public class JobManager implements IJobM
           if (Logging.perf.isDebugEnabled())
             Logging.perf.debug(" Bin "+binNames[j]+" has no more available fetches - excluding");
 
-          Object o = row.getValue(JobQueue.prioritySetField);
-          if (o == null || ((Long)o).longValue() <= prioritizationTime)
+          //???
+          //Object o = row.getValue(JobQueue.prioritySetField);
+          //if (o == null || ((Long)o).longValue() <= prioritizationTime)
+          // Fully enabling blocking document logic, for now.
+          if (true)
           {
             // Need to add a document descriptor based on this row to the blockingDocuments object!
             // This will cause it to be reprioritized preferentially, getting it out of the way if it shouldn't
@@ -9799,12 +9873,10 @@ public class JobManager implements IJobM
         return false;
 
       // If the number of chunks exceeds n, we are done
-      Iterator iter = jobQueueHash.keySet().iterator();
       int count = 0;
-      while (iter.hasNext())
+      for (Long jobID : jobQueueHash.keySet())
       {
-        Long jobID = (Long)iter.next();
-        QueueHashItem item = (QueueHashItem)jobQueueHash.get(jobID);
+        QueueHashItem item = jobQueueHash.get(jobID);
         count += item.getChunkCount();
         if (count > n)
           return false;
@@ -9821,7 +9893,7 @@ public class JobManager implements IJobM
   protected static class QueueHashItem
   {
     // The number of items per chunk for this job
-    int itemsPerChunk;
+    final int itemsPerChunk;
     // The number of chunks so far, INCLUDING incomplete chunks
     int chunkCount = 0;
     // The number of documents in the current incomplete chunk
@@ -9873,10 +9945,10 @@ public class JobManager implements IJobM
   protected static class ThrottleJobItem
   {
     /** These are the bin limits.  This is an array of ThrottleLimitSpec objects. */
-    protected ArrayList throttleLimits = new ArrayList();
+    protected List<ThrottleLimitSpec> throttleLimits = new ArrayList<ThrottleLimitSpec>();
     /** This is a map of the bins and their current counts. If an entry doesn't exist, it's considered to be
     * the same as maxBinCount. */
-    protected HashMap binCounts = new HashMap();
+    protected final Map<String,MutableInteger> binCounts = new HashMap<String,MutableInteger>();
 
     /** Constructor. */
     public ThrottleJobItem()
@@ -9906,11 +9978,9 @@ public class JobManager implements IJobM
     {
       ThrottleJobItem rval = new ThrottleJobItem();
       rval.throttleLimits = throttleLimits;
-      Iterator iter = binCounts.keySet().iterator();
-      while (iter.hasNext())
+      for (String key : binCounts.keySet())
       {
-        String key = (String)iter.next();
-        this.binCounts.put(key,((MutableInteger)binCounts.get(key)).duplicate());
+        this.binCounts.put(key,binCounts.get(key).duplicate());
       }
       return rval;
     }
@@ -9921,7 +9991,7 @@ public class JobManager implements IJobM
     */
     public boolean isEmpty(String binName)
     {
-      MutableInteger value = (MutableInteger)binCounts.get(binName);
+      MutableInteger value = binCounts.get(binName);
       int remaining;
       if (value == null)
       {
@@ -9940,7 +10010,7 @@ public class JobManager implements IJobM
     */
     public void decrement(String binName)
     {
-      MutableInteger value = (MutableInteger)binCounts.get(binName);
+      MutableInteger value = binCounts.get(binName);
       if (value == null)
       {
         int x = findMaxCount(binName);
@@ -9985,7 +10055,7 @@ public class JobManager implements IJobM
       int i = 0;
       while (i < throttleLimits.size())
       {
-        ThrottleLimitSpec spec = (ThrottleLimitSpec)throttleLimits.get(i++);
+        ThrottleLimitSpec spec = throttleLimits.get(i++);
         Pattern p = spec.getRegexp();
         Matcher m = p.matcher(binName);
         if (m.find())
@@ -10004,9 +10074,9 @@ public class JobManager implements IJobM
   protected static class ThrottleLimitSpec
   {
     /** Regexp */
-    protected Pattern regexp;
+    protected final Pattern regexp;
     /** The fetch limit for all bins matching that regexp */
-    protected int maxCount;
+    protected final int maxCount;
 
     /** Constructor */
     public ThrottleLimitSpec(String regexp, int maxCount)

Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java Tue Nov 18 14:30:38 2014
@@ -41,10 +41,11 @@ import java.util.*;
  * <tr><td>status</td><td>CHAR(1)</td><td></td></tr>
  * <tr><td>isseed</td><td>CHAR(1)</td><td></td></tr>
  * <tr><td>docpriority</td><td>FLOAT</td><td></td></tr>
- * <tr><td>priorityset</td><td>BIGINT</td><td></td></tr>
  * <tr><td>checkaction</td><td>CHAR(1)</td><td></td></tr>
  * <tr><td>processid</td><td>VARCHAR(16)</td><td></td></tr>
  * <tr><td>seedingprocessid</td><td>VARCHAR(16)</td><td></td></tr>
+ * <tr><td>needpriority</td><td>CHAR(1)</td><td></td></tr>
+ * <tr><td>needpriorityprocessid</td><td>VARCHAR(16)</td><td></td></tr>
  * </table>
  * <br><br>
  * 
@@ -77,6 +78,11 @@ public class JobQueue extends org.apache
   public final static int ACTION_RESCAN = 0;
   public final static int ACTION_REMOVE = 1;
 
+  // Need priority status
+  public final static int NEEDPRIORITY_FALSE = 0;
+  public final static int NEEDPRIORITY_INPROGRESS = 1;
+  public final static int NEEDPRIORITY_TRUE = 2;
+  
   // State descriptions are as follows:
   // PENDING means a newly-added reference that has not been scanned before.
   // ACTIVE means a newly-added reference that is being scanned for the first time.
@@ -114,19 +120,19 @@ public class JobQueue extends org.apache
   public static final String failCountField = "failcount";
   public static final String isSeedField = "isseed";
   public static final String docPriorityField = "docpriority";
-  public static final String prioritySetField = "priorityset";
   public static final String checkActionField = "checkaction";
   public static final String processIDField = "processid";
   public static final String seedingProcessIDField = "seedingprocessid";
+  public static final String needPriorityField = "needpriority";
+  public static final String needPriorityProcessIDField = "needpriorityprocessid";
   
   public static final double noDocPriorityValue = 1e9;
   public static final Double nullDocPriority = new Double(noDocPriorityValue + 1.0);
   
-  protected static Map statusMap;
+  protected static final Map<String,Integer> statusMap = new HashMap<String,Integer>();
 
   static
   {
-    statusMap = new HashMap();
     statusMap.put("P",new Integer(STATUS_PENDING));
     statusMap.put("A",new Integer(STATUS_ACTIVE));
     statusMap.put("C",new Integer(STATUS_COMPLETE));
@@ -142,25 +148,32 @@ public class JobQueue extends org.apache
     statusMap.put("H",new Integer(STATUS_HOPCOUNTREMOVED));
   }
 
-  protected static Map seedstatusMap;
+  protected static final Map<String,Integer> seedstatusMap = new HashMap<String,Integer>();
 
   static
   {
-    seedstatusMap = new HashMap();
     seedstatusMap.put("F",new Integer(SEEDSTATUS_NOTSEED));
     seedstatusMap.put("S",new Integer(SEEDSTATUS_SEED));
     seedstatusMap.put("N",new Integer(SEEDSTATUS_NEWSEED));
   }
 
-  protected static Map actionMap;
+  protected static final Map<String,Integer> actionMap = new HashMap<String,Integer>();
 
   static
   {
-    actionMap = new HashMap();
     actionMap.put("R",new Integer(ACTION_RESCAN));
     actionMap.put("D",new Integer(ACTION_REMOVE));
   }
 
+  protected static final Map<String,Integer> needPriorityMap = new HashMap<String,Integer>();
+  
+  static
+  {
+    needPriorityMap.put("T",new Integer(NEEDPRIORITY_TRUE));
+    needPriorityMap.put("I",new Integer(NEEDPRIORITY_INPROGRESS));
+    needPriorityMap.put("F",new Integer(NEEDPRIORITY_FALSE));
+  }
+  
   /** Prerequisite event manager */
   protected PrereqEventManager prereqEventManager;
 
@@ -205,23 +218,20 @@ public class JobQueue extends org.apache
         map.put(statusField,new ColumnDescription("CHAR(1)",false,false,null,null,false));
         map.put(isSeedField,new ColumnDescription("CHAR(1)",false,true,null,null,false));
         map.put(docPriorityField,new ColumnDescription("FLOAT",false,true,null,null,false));
-        map.put(prioritySetField,new ColumnDescription("BIGINT",false,true,null,null,false));
         map.put(checkActionField,new ColumnDescription("CHAR(1)",false,true,null,null,false));
         map.put(processIDField,new ColumnDescription("VARCHAR(16)",false,true,null,null,false));
         map.put(seedingProcessIDField,new ColumnDescription("VARCHAR(16)",false,true,null,null,false));
+        map.put(needPriorityField,new ColumnDescription("CHAR(1)",false,true,null,null,false));
+        map.put(needPriorityProcessIDField,new ColumnDescription("VARCHAR(16)",false,true,null,null,false));
         performCreate(map,null);
       }
       else
       {
-        // Upgrade; null docpriority fields bashed to 'infinity', so they don't slow down MySQL
+        // Upgrade; null docpriority fields bashed to 'infinity', so they don't slow down the docpriority index
         Map map = new HashMap();
         map.put(docPriorityField,nullDocPriority);
         performUpdate(map,"WHERE "+docPriorityField+" IS NULL",null,null);
         
-        map = new HashMap();
-        map.put(prioritySetField,new Long(0L));
-        performUpdate(map,"WHERE "+prioritySetField+" IS NULL",null,null);
-        
         // Also, add processIDField
         if (existing.get(processIDField) == null)
         {
@@ -237,6 +247,92 @@ public class JobQueue extends org.apache
           insertMap.put(seedingProcessIDField,new ColumnDescription("VARCHAR(16)",false,true,null,null,false));
           performAlter(insertMap,null,null,null);
         }
+        
+        if (existing.get(needPriorityField) == null && existing.get(needPriorityProcessIDField) == null)
+        {
+          Map insertMap = new HashMap();
+          insertMap.put(needPriorityField,new ColumnDescription("CHAR(1)",false,true,null,null,false));
+          insertMap.put(needPriorityProcessIDField,new ColumnDescription("VARCHAR(16)",false,true,null,null,false));
+          performAlter(insertMap,null,null,null);
+          
+          // Add the index now, because we need it for the upgrade
+          performAddIndex(null,new IndexDescription(false,new String[]{needPriorityField}));
+        }
+        
+        // Upgrade: If there's a priorityset field, remove it
+        if (existing.get("priorityset") != null)
+        {
+          List deleteList = new ArrayList();
+          deleteList.add("priorityset");
+          performAlter(null,null,deleteList,null);
+        }
+        
+        // Read all jobs and their statuses
+        IResultSet jobStatusResults = performQuery("SELECT "+Jobs.idField+","+Jobs.statusField+" FROM jobs",null,null,null);
+        Map<Long,Integer> statusMap = new HashMap<Long,Integer>();
+        for (int i = 0; i < jobStatusResults.getRowCount(); i++)
+        {
+          IResultRow row = jobStatusResults.getRow(i);
+          statusMap.put((Long)row.getValue(Jobs.idField),new Integer(Jobs.stringToStatus((String)row.getValue(Jobs.statusField))));
+        }
+          
+        // The needPriority field should be set only for documents that are active.
+        // And we need to work in small chunks for memory reasons.
+        while (true)
+        {
+          ArrayList params = new ArrayList();
+          IResultSet set = performQuery("SELECT "+idField+","+statusField+","+jobIDField+" FROM "+getTableName()+" WHERE "+
+            buildConjunctionClause(params,new ClauseDescription[]{
+              new NullCheckClause(needPriorityField,true)})+
+            " "+constructOffsetLimitClause(0,10000),
+            params,null,null);
+            
+          if (set.getRowCount() == 0)
+            break;
+          for (int i = 0 ; i < set.getRowCount() ; i++)
+          {
+            IResultRow row = set.getRow(i);
+            Long rowID = (Long)row.getValue(idField);
+            int status = stringToStatus((String)row.getValue(statusField));
+            Long jobID = (Long)row.getValue(jobIDField);
+            int jobStatus = statusMap.get(jobID).intValue();
+              
+            int needsPriority = NEEDPRIORITY_FALSE;
+            if ((status == STATUS_PENDING ||
+              status == STATUS_ACTIVE ||
+              status == STATUS_PENDINGPURGATORY || 
+              status == STATUS_ACTIVEPURGATORY ||
+              status == STATUS_ACTIVENEEDRESCAN ||
+              status == STATUS_ACTIVENEEDRESCANPURGATORY ||
+              status == STATUS_HOPCOUNTREMOVED) &&
+              (jobStatus == Jobs.STATUS_ACTIVE ||
+              jobStatus == Jobs.STATUS_ACTIVESEEDING ||
+              jobStatus == Jobs.STATUS_ACTIVEWAITING ||
+              jobStatus == Jobs.STATUS_ACTIVEWAITINGSEEDING ||
+              jobStatus == Jobs.STATUS_PAUSING ||
+              jobStatus == Jobs.STATUS_PAUSINGSEEDING ||
+              jobStatus == Jobs.STATUS_PAUSINGWAITING ||
+              jobStatus == Jobs.STATUS_PAUSINGWAITINGSEEDING ||
+              jobStatus == Jobs.STATUS_STARTINGUP ||
+              jobStatus == Jobs.STATUS_STARTINGUPMINIMAL ||
+              jobStatus == Jobs.STATUS_ACTIVE_UNINSTALLED ||
+              jobStatus == Jobs.STATUS_ACTIVESEEDING_UNINSTALLED ||
+              jobStatus == Jobs.STATUS_ACTIVE_NOOUTPUT ||
+              jobStatus == Jobs.STATUS_ACTIVESEEDING_NOOUTPUT ||
+              jobStatus == Jobs.STATUS_ACTIVE_NEITHER ||
+              jobStatus == Jobs.STATUS_ACTIVESEEDING_NEITHER))
+              needsPriority = NEEDPRIORITY_TRUE;
+                
+            Map updateMap = new HashMap();
+            updateMap.put(needPriorityField,needPriorityToString(needsPriority));
+            ArrayList updateParams = new ArrayList();
+            updateParams.add(rowID);
+            performUpdate(updateMap,"WHERE "+idField+"=?",updateParams,null);
+
+          }
+        }
+        
+
       }
 
       // Secondary table installation
@@ -248,7 +344,7 @@ public class JobQueue extends org.apache
       IndexDescription jobSeedIndex = new IndexDescription(false,new String[]{isSeedField,jobIDField});
       IndexDescription failTimeIndex = new IndexDescription(false,new String[]{failTimeField,jobIDField});
       IndexDescription actionTimeStatusIndex = new IndexDescription(false,new String[]{statusField,checkActionField,checkTimeField});
-      IndexDescription prioritysetStatusIndex = new IndexDescription(false,new String[]{statusField,prioritySetField});
+      IndexDescription needPriorityIndex = new IndexDescription(false,new String[]{needPriorityField});
       // No evidence that the extra fields help at all, for any database...
       IndexDescription docpriorityIndex = new IndexDescription(false,new String[]{docPriorityField,statusField,checkActionField,checkTimeField});
 
@@ -262,6 +358,8 @@ public class JobQueue extends org.apache
 
         if (uniqueIndex != null && id.equals(uniqueIndex))
           uniqueIndex = null;
+        else if (needPriorityIndex != null && id.equals(needPriorityIndex))
+          needPriorityIndex = null;
         else if (jobStatusIndex != null && id.equals(jobStatusIndex))
           jobStatusIndex = null;
         else if (jobSeedIndex != null && id.equals(jobSeedIndex))
@@ -270,8 +368,6 @@ public class JobQueue extends org.apache
           failTimeIndex = null;
         else if (actionTimeStatusIndex != null && id.equals(actionTimeStatusIndex))
           actionTimeStatusIndex = null;
-        else if (prioritysetStatusIndex != null && id.equals(prioritysetStatusIndex))
-          prioritysetStatusIndex = null;
         else if (docpriorityIndex != null && id.equals(docpriorityIndex))
           docpriorityIndex = null;
         else if (indexName.indexOf("_pkey") == -1)
@@ -284,6 +380,9 @@ public class JobQueue extends org.apache
       if (jobStatusIndex != null)
         performAddIndex(null,jobStatusIndex);
 
+      if (needPriorityIndex != null)
+        performAddIndex(null,needPriorityIndex);
+      
       if (jobSeedIndex != null)
         performAddIndex(null,jobSeedIndex);
 
@@ -293,9 +392,6 @@ public class JobQueue extends org.apache
       if (actionTimeStatusIndex != null)
         performAddIndex(null,actionTimeStatusIndex);
 
-      if (prioritysetStatusIndex != null)
-        performAddIndex(null,prioritysetStatusIndex);
-
       if (docpriorityIndex != null)
         performAddIndex(null,docpriorityIndex);
 
@@ -366,12 +462,22 @@ public class JobQueue extends org.apache
   public void restart(String processID)
     throws ManifoldCFException
   {
-    // Map ACTIVE back to PENDING.
+    // Map NEEDPRIORITY_INPROCESS back to NEEDPRIORITY_TRUE
     HashMap map = new HashMap();
-    map.put(statusField,statusToString(STATUS_PENDING));
-    map.put(processIDField,null);
     ArrayList list = new ArrayList();
+    map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_TRUE));
+    map.put(needPriorityProcessIDField,null);
     String query = buildConjunctionClause(list,new ClauseDescription[]{
+      new UnitaryClause(needPriorityField,needPriorityToString(NEEDPRIORITY_INPROGRESS)),
+      new UnitaryClause(processIDField,processID)});
+    performUpdate(map,"WHERE "+query,list,null);
+
+    // Map ACTIVE back to PENDING.
+    map.clear();
+    list.clear();
+    map.put(statusField,statusToString(STATUS_PENDING));
+    map.put(processIDField,null);
+    query = buildConjunctionClause(list,new ClauseDescription[]{
       new MultiClause(statusField,new Object[]{
         statusToString(STATUS_ACTIVE),
         statusToString(STATUS_ACTIVENEEDRESCAN)}),
@@ -423,7 +529,6 @@ public class JobQueue extends org.apache
     // values of the reindexing parameters will cause reindexing to occur at this point, but users
     // can configure them higher and shut this down.
     noteModifications(0,50000,0);
-    // Always analyze.
     unconditionallyAnalyzeTables();
 
     TrackerClass.noteGlobalChange("Restart");
@@ -434,12 +539,21 @@ public class JobQueue extends org.apache
   public void restart()
     throws ManifoldCFException
   {
-    // Map ACTIVE back to PENDING.
+    // Map NEEDPRIORITY_INPROGRESS to NEEDPRIORITY_TRUE
     HashMap map = new HashMap();
-    map.put(statusField,statusToString(STATUS_PENDING));
-    map.put(processIDField,null);
     ArrayList list = new ArrayList();
+    map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_TRUE));
+    map.put(needPriorityProcessIDField,null);
     String query = buildConjunctionClause(list,new ClauseDescription[]{
+      new UnitaryClause(needPriorityField,needPriorityToString(NEEDPRIORITY_INPROGRESS))});
+    performUpdate(map,"WHERE "+query,list,null);
+
+    // Map ACTIVE back to PENDING.
+    map.clear();
+    list.clear();
+    map.put(statusField,statusToString(STATUS_PENDING));
+    map.put(processIDField,null);
+    query = buildConjunctionClause(list,new ClauseDescription[]{
       new MultiClause(statusField,new Object[]{
         statusToString(STATUS_ACTIVE),
         statusToString(STATUS_ACTIVENEEDRESCAN)})});
@@ -761,7 +875,7 @@ public class JobQueue extends org.apache
     // Map COMPLETE to PENDINGPURGATORY
     HashMap map = new HashMap();
     map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
-    map.put(prioritySetField,new Long(0L));
+    map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_TRUE));
     // Do not reset priorities here!  They should all be blank at this point.
     map.put(checkTimeField,new Long(0L));
     map.put(checkActionField,actionToString(ACTION_RESCAN));
@@ -819,7 +933,7 @@ public class JobQueue extends org.apache
     // Map COMPLETE to PENDINGPURGATORY.
     HashMap map = new HashMap();
     map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
-    map.put(prioritySetField,new Long(0L));
+    map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_TRUE));
     // Do not reset priorities here!  They should all be blank at this point.
     map.put(checkTimeField,new Long(0L));
     map.put(checkActionField,actionToString(ACTION_RESCAN));
@@ -893,12 +1007,27 @@ public class JobQueue extends org.apache
     noteModifications(0,0,1);
   }
 
+  /** Prepare to calculate a document priority for a given row. */
+  public void markNeedPriorityInProgress(Long rowID, String processID)
+    throws ManifoldCFException
+  {
+    HashMap map = new HashMap();
+    map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_INPROGRESS));
+    map.put(needPriorityProcessIDField,processID);
+    ArrayList list = new ArrayList();
+    String query = buildConjunctionClause(list,new ClauseDescription[]{
+      new UnitaryClause(idField,rowID)});
+    performUpdate(map,"WHERE "+query,list,null);
+    noteModifications(0,1,0);
+  }
+  
   /** Write out a document priority */
-  public void writeDocPriority(long currentTime, Long rowID, IPriorityCalculator priority)
+  public void writeDocPriority(Long rowID, IPriorityCalculator priority)
     throws ManifoldCFException
   {
     HashMap map = new HashMap();
-    map.put(prioritySetField,new Long(currentTime));
+    map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
+    map.put(needPriorityProcessIDField,null);
     map.put(docPriorityField,new Double(priority.getDocumentPriority()));
     ArrayList list = new ArrayList();
     String query = buildConjunctionClause(list,new ClauseDescription[]{
@@ -907,12 +1036,13 @@ public class JobQueue extends org.apache
     noteModifications(0,1,0);
   }
 
-  /** Clear all document priorities for a job */
+  /** Clear all document priorities for a job that is going to sleep */
   public void clearDocPriorities(Long jobID)
     throws ManifoldCFException
   {
     HashMap map = new HashMap();
-    map.put(prioritySetField,new Long(0L));
+    map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
+    map.put(needPriorityProcessIDField,null);
     map.put(docPriorityField,nullDocPriority);
     ArrayList list = new ArrayList();
     String query = buildConjunctionClause(list,new ClauseDescription[]{
@@ -921,6 +1051,22 @@ public class JobQueue extends org.apache
     noteModifications(0,1,0);
   }
   
+  /** Clear all document priorities globally for all documents that
+  * have priorities set, and signal that we need new priorities for all.
+  */
+  public void clearAllDocPriorities()
+    throws ManifoldCFException
+  {
+    HashMap map = new HashMap();
+    map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_TRUE));
+    map.put(docPriorityField,nullDocPriority);
+    ArrayList list = new ArrayList();
+    String query = buildConjunctionClause(list,new ClauseDescription[]{
+      new UnitaryClause(docPriorityField,"<",nullDocPriority)});
+    performUpdate(map,"WHERE "+query,list,null);
+    noteModifications(0,1,0);
+  }
+  
   /** Set the "completed" status for a record.
   */
   public void updateCompletedRecord(Long recID, int currentStatus)
@@ -941,7 +1087,7 @@ public class JobQueue extends org.apache
       checkTimeValue = null;
       // Remove document priority; we don't want to pollute the queue.  See CONNECTORS-290.
       map.put(docPriorityField,nullDocPriority);
-      map.put(prioritySetField,new Long(0L));
+      map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
       break;
     case STATUS_ACTIVENEEDRESCAN:
     case STATUS_ACTIVENEEDRESCANPURGATORY:
@@ -1214,7 +1360,7 @@ public class JobQueue extends org.apache
   * The record is presumed to exist and have been locked, via "FOR UPDATE".
   */
   public void updateExistingRecordInitial(Long recordID, int currentStatus, Long checkTimeValue,
-    long desiredExecuteTime, long currentTime, IPriorityCalculator desiredPriority, String[] prereqEvents,
+    long desiredExecuteTime, IPriorityCalculator desiredPriority, String[] prereqEvents,
     String processID)
     throws ManifoldCFException
   {
@@ -1253,7 +1399,7 @@ public class JobQueue extends org.apache
       map.put(failCountField,null);
       // Update the doc priority.
       map.put(docPriorityField,new Double(desiredPriority.getDocumentPriority()));
-      map.put(prioritySetField,new Long(currentTime));
+      map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
       break;
 
     case STATUS_PENDING:
@@ -1310,7 +1456,7 @@ public class JobQueue extends org.apache
   *@param docID is the local document identifier.
   */
   public void insertNewRecordInitial(Long jobID, String docHash, String docID, IPriorityCalculator desiredDocPriority,
-    long desiredExecuteTime, long currentTime, String[] prereqEvents, String processID)
+    long desiredExecuteTime, String[] prereqEvents, String processID)
     throws ManifoldCFException
   {
     // No prerequisites should be possible at this point.
@@ -1330,7 +1476,7 @@ public class JobQueue extends org.apache
     map.put(seedingProcessIDField,processID);
     // Set the document priority
     map.put(docPriorityField,new Double(desiredDocPriority.getDocumentPriority()));
-    map.put(prioritySetField,new Long(currentTime));
+    map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
     performInsert(map,null);
     prereqEventManager.addRows(recordID,prereqEvents);
     noteModifications(1,0,0);
@@ -1511,7 +1657,7 @@ public class JobQueue extends org.apache
   * The record is presumed to exist and have been locked, via "FOR UPDATE".
   */
   public void updateExistingRecord(Long recordID, int currentStatus, Long checkTimeValue,
-    long desiredExecuteTime, long currentTime, boolean otherChangesSeen,
+    long desiredExecuteTime, boolean otherChangesSeen,
     IPriorityCalculator desiredPriority, String[] prereqEvents)
     throws ManifoldCFException
   {
@@ -1529,7 +1675,7 @@ public class JobQueue extends org.apache
       map.put(failCountField,null);
       // Going into pending: set the docpriority.
       map.put(docPriorityField,new Double(desiredPriority.getDocumentPriority()));
-      map.put(prioritySetField,new Long(currentTime));
+      map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
       break;
     case STATUS_COMPLETE:
     case STATUS_BEINGCLEANED:
@@ -1546,7 +1692,7 @@ public class JobQueue extends org.apache
         map.put(failCountField,null);
         // Going into pending: set the docpriority.
         map.put(docPriorityField,new Double(desiredPriority.getDocumentPriority()));
-        map.put(prioritySetField,new Long(currentTime));
+        map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
         break;
       }
       return;
@@ -1575,7 +1721,7 @@ public class JobQueue extends org.apache
         map.put(failCountField,null);
         // Going into pending: set the docpriority.
         map.put(docPriorityField,new Double(desiredPriority.getDocumentPriority()));
-        map.put(prioritySetField,new Long(currentTime));
+        map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
         break;
       }
       return;
@@ -1599,7 +1745,7 @@ public class JobQueue extends org.apache
         map.put(failCountField,null);
         // Going into pending: set the docpriority.
         map.put(docPriorityField,new Double(desiredPriority.getDocumentPriority()));
-        map.put(prioritySetField,new Long(currentTime));
+        map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
         break;
       }
       return;
@@ -1641,7 +1787,7 @@ public class JobQueue extends org.apache
   *
   */
   public void insertNewRecord(Long jobID, String docIDHash, String docID, IPriorityCalculator desiredDocPriority, long desiredExecuteTime,
-    long currentTime, String[] prereqEvents)
+    String[] prereqEvents)
     throws ManifoldCFException
   {
     HashMap map = new HashMap();
@@ -1655,7 +1801,7 @@ public class JobQueue extends org.apache
     map.put(statusField,statusToString(STATUS_PENDING));
     // Be sure to set the priority also
     map.put(docPriorityField,new Double(desiredDocPriority.getDocumentPriority()));
-    map.put(prioritySetField,new Long(currentTime));
+    map.put(needPriorityField,needPriorityToString(NEEDPRIORITY_FALSE));
     performInsert(map,null);
     prereqEventManager.addRows(recordID,prereqEvents);
     noteModifications(1,0,0);
@@ -1690,7 +1836,7 @@ public class JobQueue extends org.apache
   {
     if (x == null || x.length() == 0)
       return SEEDSTATUS_NOTSEED;
-    Integer y = (Integer)seedstatusMap.get(x);
+    Integer y = seedstatusMap.get(x);
     if (y == null)
       throw new ManifoldCFException("Unknown seed status code: "+x);
     return y.intValue();
@@ -1701,7 +1847,7 @@ public class JobQueue extends org.apache
   public static int stringToAction(String value)
     throws ManifoldCFException
   {
-    Integer x = (Integer)actionMap.get(value);
+    Integer x = actionMap.get(value);
     if (x == null)
       throw new ManifoldCFException("Unknown action string: '"+value+"'");
     return x.intValue();
@@ -1722,6 +1868,37 @@ public class JobQueue extends org.apache
     }
   }
 
+  /** Convert need priority value to boolean.
+  */
+  public static int stringToNeedPriority(String value)
+    throws ManifoldCFException
+  {
+    if (value == null || value.length() == 0)
+      return NEEDPRIORITY_FALSE;
+    Integer x = needPriorityMap.get(value);
+    if (x == null)
+      throw new ManifoldCFException("Unknown needpriority string: '"+value+"'");
+    return x.intValue();
+  }
+  
+  /** Convert boolean to need priority value.
+  */
+  public static String needPriorityToString(int value)
+    throws ManifoldCFException
+  {
+    switch (value)
+    {
+    case NEEDPRIORITY_TRUE:
+      return "T";
+    case NEEDPRIORITY_FALSE:
+      return "F";
+    case NEEDPRIORITY_INPROGRESS:
+      return "I";
+    default:
+      throw new ManifoldCFException("Bad needpriority value: "+Integer.toString(value));
+    }
+  }
+  
   /** Convert status field value to integer.
   *@param value is the string.
   *@return the integer.
@@ -1729,7 +1906,7 @@ public class JobQueue extends org.apache
   public static int stringToStatus(String value)
     throws ManifoldCFException
   {
-    Integer x = (Integer)statusMap.get(value);
+    Integer x = statusMap.get(value);
     if (x == null)
       throw new ManifoldCFException("Unknown status string: '"+value+"'");
     return x.intValue();

Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/reprioritizationtracker/ReprioritizationTracker.java Tue Nov 18 14:30:38 2014
@@ -39,7 +39,6 @@ public class ReprioritizationTracker imp
   protected final static String trackerWriteLock = "_REPR_TRACKER_LOCK_";
   protected final static String trackerProcessIDResource = "_REPR_TRACKER_PID_";
   protected final static String trackerReproIDResource = "_REPR_TRACKER_RID_";
-  protected final static String trackerTimestampResource = "_REPR_TIMESTAMP_";
   protected final static String trackerMinimumDepthResource = "_REPR_MINDEPTH_";
   
   /** Lock manager */
@@ -67,21 +66,18 @@ public class ReprioritizationTracker imp
   *@param reproID is the reprocessing thread ID
   */
   @Override
-  public void startReprioritization(long prioritizationTime, String processID, String reproID)
+  public void startReprioritization(String processID, String reproID)
     throws ManifoldCFException
   {
     lockManager.enterWriteLock(trackerWriteLock);
     try
     {
-      Long currentTime = readTime();
       String currentProcessID = readProcessID();
-      if (currentTime != null && currentProcessID != null)
+      if (currentProcessID != null)
       {
         // Already a reprioritization in progress.
-        if (prioritizationTime <= currentTime.longValue())
-          return;
+        return;
       }
-      writeTime(new Long(prioritizationTime));
       writeProcessID(processID);
       writeReproID(reproID);
       try
@@ -90,7 +86,6 @@ public class ReprioritizationTracker imp
       }
       catch (Throwable e)
       {
-        writeTime(null);
         writeProcessID(null);
         writeReproID(null);
         if (e instanceof Error)
@@ -111,30 +106,6 @@ public class ReprioritizationTracker imp
   }
   
   
-  /** Retrieve the current reprioritization time stamp.  This should be obtained before
-  * performing any prioritization steps.
-  *@return the current prioritization timestamp, or null if no prioritization is in effect.
-  */
-  @Override
-  public Long checkReprioritizationInProgress()
-    throws ManifoldCFException
-  {
-    lockManager.enterWriteLock(trackerWriteLock);
-    try
-    {
-      Long currentTime = readTime();
-      String currentProcessID = readProcessID();
-      String currentReproID = readReproID();
-      if (currentTime == null || currentProcessID == null || currentReproID == null)
-        return null;
-      return currentTime;
-    }
-    finally
-    {
-      lockManager.leaveWriteLock(trackerWriteLock);
-    }
-  }
-
   /** Complete a reprioritization activity.  Prioritization will be marked as complete
   * only if the processID matches the one that started the current reprioritization.
   *@param processID is the process ID of the process completing the prioritization.
@@ -146,13 +117,11 @@ public class ReprioritizationTracker imp
     lockManager.enterWriteLock(trackerWriteLock);
     try
     {
-      Long currentTime = readTime();
       String currentProcessID = readProcessID();
       String currentReproID = readReproID();
-      if (currentTime != null && currentProcessID != null && currentReproID != null && currentReproID.equals(reproID))
+      if (currentProcessID != null && currentReproID != null && currentReproID.equals(reproID))
       {
         // Null out the fields
-        writeTime(null);
         writeProcessID(null);
         writeReproID(null);
       }
@@ -174,10 +143,9 @@ public class ReprioritizationTracker imp
     lockManager.enterWriteLock(trackerWriteLock);
     try
     {
-      Long currentTime = readTime();
       String currentProcessID = readProcessID();
       String currentReproID = readReproID();
-      if (currentTime != null && currentProcessID != null && currentReproID != null && currentProcessID.equals(processID))
+      if (currentProcessID != null && currentReproID != null && currentProcessID.equals(processID))
         return currentReproID;
       return null;
     }
@@ -211,9 +179,8 @@ public class ReprioritizationTracker imp
       lockManager.enterWriteLock(trackerWriteLock);
       try
       {
-        Long reproTime = readTime();
         String processID = readProcessID();
-        if (reproTime == null || processID == null)
+        if (processID == null)
         {
           double currentMinimumDepth = readMinimumDepth();
 
@@ -330,55 +297,6 @@ public class ReprioritizationTracker imp
   
   // Protected methods
   
-  /** Read time.
-  *@return the time, or null if none.
-  */
-  protected Long readTime()
-    throws ManifoldCFException
-  {
-    byte[] timeData = lockManager.readData(trackerTimestampResource);
-    if (timeData == null || timeData.length != 8)
-    {
-      return null;
-    }
-    
-    long rval = (((long)timeData[0]) & 0xffL) +
-      ((((long)timeData[1]) << 8) & 0xff00L) +
-      ((((long)timeData[2]) << 16) & 0xff0000L) +
-      ((((long)timeData[3]) << 24) & 0xff000000L) +
-      ((((long)timeData[4]) << 32) & 0xff00000000L) +
-      ((((long)timeData[5]) << 40) & 0xff0000000000L) +
-      ((((long)timeData[6]) << 48) & 0xff000000000000L) +
-      ((((long)timeData[7]) << 56) & 0xff00000000000000L);
-    
-    return new Long(rval);
-  }
-  
-  /** Write time.
-  *@param time is the time to write.
-  */
-  protected void writeTime(Long timeValue)
-    throws ManifoldCFException
-  {
-    if (timeValue == null)
-      lockManager.writeData(trackerTimestampResource, null);
-    else
-    {
-      long time = timeValue.longValue();
-      byte[] timeData = new byte[8];
-      timeData[0] = (byte)(time & 0xffL);
-      timeData[1] = (byte)((time >> 8) & 0xffL);
-      timeData[2] = (byte)((time >> 16) & 0xffL);
-      timeData[3] = (byte)((time >> 24) & 0xffL);
-      timeData[4] = (byte)((time >> 32) & 0xffL);
-      timeData[5] = (byte)((time >> 40) & 0xffL);
-      timeData[6] = (byte)((time >> 48) & 0xffL);
-      timeData[7] = (byte)((time >> 56) & 0xffL);
-      
-      lockManager.writeData(trackerTimestampResource, timeData);
-    }
-  }
-  
   /** Read process ID.
   *@return processID, or null if none.
   */

Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/CrawlerAgent.java Tue Nov 18 14:30:38 2014
@@ -153,7 +153,9 @@ public class CrawlerAgent implements IAg
     // to take care of are dangling ones that won't get done because the process that was doing them went
     // away.  BUT: somebody may have blown away lock info, in which case we won't know anything at all.
     // So we do everything in that case.
-    ManifoldCF.resetAllDocumentPriorities(threadContext,System.currentTimeMillis(),currentProcessID);
+    
+    ManifoldCF.resetAllDocumentPriorities(threadContext,currentProcessID);
+
   }
   
   /** Cleanup after agents process.
@@ -170,12 +172,23 @@ public class CrawlerAgent implements IAg
   {
     IJobManager jobManager = JobManagerFactory.make(threadContext);
     jobManager.cleanupProcessData(cleanupProcessID);
+    
+    // If one agents process was starting a reprioritization, it could have started the reprioritization sequence, but
+    // failed to complete it.  If so, we may need to reset/complete the reprioritization sequence, which is defined as:
+    // - Resetting prioritization parameters
+    // - Removing all existing document priorities
+    // These must go together in order for the reset to be correct.
+    
     IReprioritizationTracker rt = ReprioritizationTrackerFactory.make(threadContext);
     String reproID = rt.isSpecifiedProcessReprioritizing(cleanupProcessID);
     if (reproID != null)
     {
       // We have to take over the prioritization for the process, which apparently died
       // in the middle.
+      
+      jobManager.clearAllDocumentPriorities();
+      
+      /*
       IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(threadContext);
 
       // Reprioritize all documents in the jobqueue, 1000 at a time
@@ -188,6 +201,7 @@ public class CrawlerAgent implements IAg
       // activity.
       // In order for this to be the correct functionality, ALL reseeding and requeuing operations MUST reset the associated document
       // priorities.
+      // ??? -- start the process of reprioritization ONLY; don't do the whole thing.
       while (true)
       {
         long startTime = System.currentTimeMillis();
@@ -200,15 +214,16 @@ public class CrawlerAgent implements IAg
         }
         long updateTime = currentTimeValue.longValue();
         
-        DocumentDescription[] docs = jobManager.getNextNotYetProcessedReprioritizationDocuments(updateTime, 10000);
+        DocumentDescription[] docs = jobManager.getNextNotYetProcessedReprioritizationDocuments(10000);
         if (docs.length == 0)
           break;
 
         // Calculate new priorities for all these documents
-        ManifoldCF.writeDocumentPriorities(threadContext,docs,connectionMap,jobDescriptionMap,updateTime);
+        ManifoldCF.writeDocumentPriorities(threadContext,docs,connectionMap,jobDescriptionMap);
 
         Logging.threads.debug("Reprioritized "+Integer.toString(docs.length)+" not-yet-processed documents in "+new Long(System.currentTimeMillis()-startTime)+" ms");
       }
+      */
       
       rt.doneReprioritization(reproID);
     }

Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobResetThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobResetThread.java?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobResetThread.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobResetThread.java Tue Nov 18 14:30:38 2014
@@ -106,7 +106,7 @@ public class JobResetThread extends Thre
           {
             Logging.threads.debug("Job reset thread reprioritizing documents...");
 
-            ManifoldCF.resetAllDocumentPriorities(threadContext,currentTime,processID);
+            ManifoldCF.resetAllDocumentPriorities(threadContext,processID);
             
             Logging.threads.debug("Job reset thread done reprioritizing documents.");
 

Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1640350&r1=1640349&r2=1640350&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java (original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java Tue Nov 18 14:30:38 2014
@@ -966,7 +966,7 @@ public class ManifoldCF extends org.apac
     }
 
     // Now, requeue the documents with the new priorities
-    jobManager.carrydownChangeDocumentMultiple(requeueCandidates,currentTime,docPriorities);
+    jobManager.carrydownChangeDocumentMultiple(requeueCandidates,docPriorities);
   }
 
   /** Stuff colons so we can't have conflicts. */
@@ -1013,11 +1013,26 @@ public class ManifoldCF extends org.apac
   /** Reset all (active) document priorities.  This operation may occur due to various externally-triggered
   * events, such a job abort, pause, resume, wait, or unwait.
   */
-  public static void resetAllDocumentPriorities(IThreadContext threadContext, long currentTime, String processID)
+  public static void resetAllDocumentPriorities(IThreadContext threadContext, String processID)
     throws ManifoldCFException
   {
-    ILockManager lockManager = LockManagerFactory.make(threadContext);
+    // The reprioritization cycle is as follows now:
+    // (1) We reset the reprioritization tracker, which causes all bins to be be reset, and locks reprioritization so that it is blocked;
+    // (2) We clear all document priorities;
+    // (3) We unlock reprioritization, so that it may proceed.
     IJobManager jobManager = JobManagerFactory.make(threadContext);
+    IReprioritizationTracker rt = ReprioritizationTrackerFactory.make(threadContext);
+
+    String reproID = IDFactory.make(threadContext);
+
+    rt.startReprioritization(processID,reproID);
+
+    jobManager.clearAllDocumentPriorities();
+
+    rt.doneReprioritization(reproID);
+
+  /*
+    ILockManager lockManager = LockManagerFactory.make(threadContext);
     IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(threadContext);
     IReprioritizationTracker rt = ReprioritizationTrackerFactory.make(threadContext);
 
@@ -1034,6 +1049,7 @@ public class ManifoldCF extends org.apac
     // activity.
     // In order for this to be the correct functionality, ALL reseeding and requeuing operations MUST reset the associated document
     // priorities.
+    // ???? Should only start the process of reprioritization, not complete it.
     while (true)
     {
       long startTime = System.currentTimeMillis();
@@ -1046,24 +1062,24 @@ public class ManifoldCF extends org.apac
       }
       long updateTime = currentTimeValue.longValue();
       
-      DocumentDescription[] docs = jobManager.getNextNotYetProcessedReprioritizationDocuments(updateTime, 10000);
+      DocumentDescription[] docs = jobManager.getNextNotYetProcessedReprioritizationDocuments(10000);
       if (docs.length == 0)
         break;
 
       // Calculate new priorities for all these documents
-      writeDocumentPriorities(threadContext,docs,connectionMap,jobDescriptionMap,updateTime);
+      writeDocumentPriorities(threadContext,docs,connectionMap,jobDescriptionMap);
 
       Logging.threads.debug("Reprioritized "+Integer.toString(docs.length)+" not-yet-processed documents in "+new Long(System.currentTimeMillis()-startTime)+" ms");
     }
     
     rt.doneReprioritization(reproID);
+    */
   }
   
   /** Write a set of document priorities, based on the current queue tracker.
   */
   public static void writeDocumentPriorities(IThreadContext threadContext, DocumentDescription[] descs,
-    Map<String,IRepositoryConnection> connectionMap, Map<Long,IJobDescription> jobDescriptionMap,
-    long currentTime)
+    Map<String,IRepositoryConnection> connectionMap, Map<Long,IJobDescription> jobDescriptionMap)
     throws ManifoldCFException
   {
     IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
@@ -1148,7 +1164,7 @@ public class ManifoldCF extends org.apac
     rt.preloadBinValues();
     
     // Now, write all the priorities we can.
-    jobManager.writeDocumentPriorities(currentTime,descs,priorities);
+    jobManager.writeDocumentPriorities(descs,priorities);
 
     rt.clearPreloadedValues();
   }