You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/08/14 12:42:17 UTC
svn commit: r1513803 - in
/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs:
JobManager.java JobQueue.java
Author: kwright
Date: Wed Aug 14 10:42:17 2013
New Revision: 1513803
URL: http://svn.apache.org/r1513803
Log:
Track purgatory condition of HOPCOUNTREMOVED records, so we don't ever treat a purgatory record like a pending
Modified:
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1513803&r1=1513802&r2=1513803&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java Wed Aug 14 10:42:17 2013
@@ -1498,6 +1498,7 @@ public class JobManager implements IJobM
.append(database.buildConjunctionClause(list,new ClauseDescription[]{
new MultiClause(jobQueue.statusField,new Object[]{
JobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVED),
+ JobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVEDPURGATORY),
JobQueue.statusToString(jobQueue.STATUS_PENDING),
JobQueue.statusToString(jobQueue.STATUS_PENDINGPURGATORY)}),
new UnitaryClause(jobQueue.prioritySetField,"<",new Long(currentTime))})).append(" AND ")
@@ -7474,6 +7475,7 @@ public class JobManager implements IJobM
.append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Being removed'")
.append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Being removed'")
.append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Out of scope'")
+ .append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Out of scope'")
.append(" ELSE 'Unknown'")
.append(" END AS state,")
.append("CASE")
@@ -7505,7 +7507,7 @@ public class JobManager implements IJobM
.append(" AND ").append("t0.").append(jobQueue.checkTimeField).append(" IS NULL")
.append(" THEN 'Waiting forever'")
.append(" WHEN ")
- .append("t0.").append(jobQueue.statusField).append("=?")
+ .append("t0.").append(jobQueue.statusField).append(" IN (?,?)")
.append(" THEN 'Hopcount exceeded'")
.append(" WHEN ").append("t0.").append(jobQueue.statusField).append(" IN (?,?,?)")
.append(" THEN 'Deleting'")
@@ -7544,6 +7546,7 @@ public class JobManager implements IJobM
list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED));
list.add(jobQueue.statusToString(jobQueue.STATUS_ELIGIBLEFORDELETE));
list.add(jobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVED));
+ list.add(jobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVEDPURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_COMPLETE));
list.add(jobQueue.statusToString(jobQueue.STATUS_UNCHANGED));
@@ -7569,6 +7572,7 @@ public class JobManager implements IJobM
list.add(jobQueue.statusToString(jobQueue.STATUS_PENDINGPURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVED));
+ list.add(jobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVEDPURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED));
list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED));
@@ -7694,7 +7698,7 @@ public class JobManager implements IJobM
.append(" as waitingforever,")
.append("CASE")
.append(" WHEN ")
- .append(jobQueue.statusField).append("=?")
+ .append(jobQueue.statusField).append(" IN (?,?)")
.append(" THEN 1 ELSE 0")
.append(" END")
.append(" as hopcountexceeded");
@@ -7740,6 +7744,7 @@ public class JobManager implements IJobM
list.add(jobQueue.statusToString(jobQueue.STATUS_PENDINGPURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVED));
+ list.add(jobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVEDPURGATORY));
addCriteria(sb,list,"",connectionName,filterCriteria,false);
sb.append(") t1 GROUP BY idbucket");
@@ -7834,7 +7839,8 @@ public class JobManager implements IJobM
case DOCSTATE_OUTOFSCOPE:
sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{
new MultiClause(fieldPrefix+jobQueue.statusField,new Object[]{
- jobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVED)})}));
+ jobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVED),
+ jobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVEDPURGATORY)})}));
break;
}
k++;
@@ -7925,7 +7931,8 @@ public class JobManager implements IJobM
case DOCSTATUS_HOPCOUNTEXCEEDED:
sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{
new MultiClause(fieldPrefix+jobQueue.statusField,new Object[]{
- jobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVED)})}));
+ jobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVED),
+ jobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVEDPURGATORY)})}));
break;
}
k++;
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java?rev=1513803&r1=1513802&r2=1513803&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java Wed Aug 14 10:42:17 2013
@@ -70,6 +70,7 @@ public class JobQueue extends org.apache
public final static int STATUS_BEINGCLEANED = 10;
public final static int STATUS_ELIGIBLEFORDELETE = 11;
public final static int STATUS_HOPCOUNTREMOVED = 12;
+ public final static int STATUS_HOPCOUNTREMOVEDPURGATORY = 13;
// Action values
public final static int ACTION_RESCAN = 0;
@@ -133,6 +134,7 @@ public class JobQueue extends org.apache
statusMap.put("f",new Integer(STATUS_ACTIVENEEDRESCANPURGATORY));
statusMap.put("d",new Integer(STATUS_BEINGCLEANED));
statusMap.put("H",new Integer(STATUS_HOPCOUNTREMOVED));
+ statusMap.put("h",new Integer(STATUS_HOPCOUNTREMOVEDPURGATORY));
}
protected static Map seedstatusMap;
@@ -409,9 +411,22 @@ public class JobQueue extends org.apache
new UnitaryClause(jobIDField,jobID),
new UnitaryClause(statusField,statusToString(STATUS_HOPCOUNTREMOVED))});
performUpdate(map,"WHERE "+query,list,null);
- unconditionallyAnalyzeTables();
-
+
TrackerClass.noteJobChange(jobID,"Map HOPCOUNTREMOVED to PENDING");
+
+ // Map HOPCOUNTREMOVEDPURGATORY to PENDINGPURGATORY
+ map.clear();
+ map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
+ map.put(checkTimeField,new Long(0L));
+ list = new ArrayList();
+ query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(jobIDField,jobID),
+ new UnitaryClause(statusField,statusToString(STATUS_HOPCOUNTREMOVEDPURGATORY))});
+ performUpdate(map,"WHERE "+query,list,null);
+
+ TrackerClass.noteJobChange(jobID,"Map HOPCOUNTREMOVEDPURGATORY to PENDINGPURGATORY");
+
+ unconditionallyAnalyzeTables();
}
@@ -538,6 +553,7 @@ public class JobQueue extends org.apache
new UnitaryClause(jobIDField,jobID),
new MultiClause(statusField,new Object[]{
statusToString(STATUS_PENDINGPURGATORY),
+ statusToString(STATUS_HOPCOUNTREMOVEDPURGATORY),
statusToString(STATUS_COMPLETE),
statusToString(STATUS_UNCHANGED),
statusToString(STATUS_PURGATORY)})});
@@ -593,6 +609,7 @@ public class JobQueue extends org.apache
new UnitaryClause(jobIDField,jobID),
new MultiClause(statusField,new Object[]{
statusToString(STATUS_PENDINGPURGATORY),
+ statusToString(STATUS_HOPCOUNTREMOVEDPURGATORY),
statusToString(STATUS_UNCHANGED),
statusToString(STATUS_COMPLETE)})});
performUpdate(map,"WHERE "+query,list,null);
@@ -836,13 +853,19 @@ public class JobQueue extends org.apache
switch (currentStatus)
{
case STATUS_ACTIVE:
- case STATUS_ACTIVEPURGATORY:
// Mark as hopcountremove
newStatus = STATUS_HOPCOUNTREMOVED;
actionFieldValue = actionToString(ACTION_RESCAN);
checkTimeValue = new Long(0L);
rval = true;
break;
+ case STATUS_ACTIVEPURGATORY:
+ // Mark as hopcountremovepurgatory
+ newStatus = STATUS_HOPCOUNTREMOVEDPURGATORY;
+ actionFieldValue = actionToString(ACTION_RESCAN);
+ checkTimeValue = new Long(0L);
+ rval = true;
+ break;
case STATUS_ACTIVENEEDRESCAN:
case STATUS_ACTIVENEEDRESCANPURGATORY:
newStatus = STATUS_PENDINGPURGATORY;
@@ -1619,6 +1642,8 @@ public class JobQueue extends org.apache
return "d";
case STATUS_HOPCOUNTREMOVED:
return "H";
+ case STATUS_HOPCOUNTREMOVEDPURGATORY:
+ return "h";
default:
throw new ManifoldCFException("Bad status value: "+Integer.toString(status));
}