You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/04/05 09:52:14 UTC
svn commit: r1464874 - in /manifoldcf/branches/CONNECTORS-590-2: ./
framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/
Author: kwright
Date: Fri Apr 5 07:52:13 2013
New Revision: 1464874
URL: http://svn.apache.org/r1464874
Log:
Port tracking code from CONNECTORS-590 to a more modern branch.
Added:
manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/TrackerClass.java
- copied unchanged from r1464871, manifoldcf/branches/CONNECTORS-590/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/TrackerClass.java
Modified:
manifoldcf/branches/CONNECTORS-590-2/ (props changed)
manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
Propchange: manifoldcf/branches/CONNECTORS-590-2/
------------------------------------------------------------------------------
Merged /manifoldcf/branches/CONNECTORS-590:r1425803-1464871
Modified: manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1464874&r1=1464873&r2=1464874&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java (original)
+++ manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java Fri Apr 5 07:52:13 2013
@@ -645,12 +645,14 @@ public class JobManager implements IJobM
// Clean up carrydown stuff
carryDown.reset();
database.performCommit();
+ TrackerClass.noteCommit();
Logging.jobs.debug("Reset complete");
break;
}
catch (ManifoldCFException e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
{
if (Logging.perf.isDebugEnabled())
@@ -663,6 +665,7 @@ public class JobManager implements IJobM
catch (Error e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
throw e;
}
finally
@@ -687,11 +690,13 @@ public class JobManager implements IJobM
{
jobQueue.resetDocumentWorkerStatus();
database.performCommit();
+ TrackerClass.noteCommit();
break;
}
catch (ManifoldCFException e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
{
if (Logging.perf.isDebugEnabled())
@@ -704,6 +709,7 @@ public class JobManager implements IJobM
catch (Error e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
throw e;
}
finally
@@ -732,6 +738,7 @@ public class JobManager implements IJobM
{
Logging.jobs.debug("Resetting doc deleting status");
jobQueue.resetDocDeleteWorkerStatus();
+ TrackerClass.noteCommit();
Logging.jobs.debug("Reset complete");
}
@@ -742,6 +749,7 @@ public class JobManager implements IJobM
{
Logging.jobs.debug("Resetting doc cleaning status");
jobQueue.resetDocCleanupWorkerStatus();
+ TrackerClass.noteCommit();
Logging.jobs.debug("Reset complete");
}
@@ -997,6 +1005,7 @@ public class JobManager implements IJobM
}
database.performCommit();
+ TrackerClass.noteCommit();
if (Logging.perf.isDebugEnabled())
Logging.perf.debug("Done pruning unindexable docs after "+new Long(System.currentTimeMillis()-startTime).toString()+" ms.");
@@ -1007,11 +1016,13 @@ public class JobManager implements IJobM
catch (Error e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
throw e;
}
catch (ManifoldCFException e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
{
if (Logging.perf.isDebugEnabled())
@@ -1258,6 +1269,7 @@ public class JobManager implements IJobM
}
database.performCommit();
+ TrackerClass.noteCommit();
if (Logging.perf.isDebugEnabled())
Logging.perf.debug("Done pruning unindexable docs after "+new Long(System.currentTimeMillis()-startTime).toString()+" ms.");
@@ -1268,11 +1280,13 @@ public class JobManager implements IJobM
catch (Error e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
throw e;
}
catch (ManifoldCFException e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
{
if (Logging.perf.isDebugEnabled())
@@ -1821,6 +1835,7 @@ public class JobManager implements IJobM
}
database.performCommit();
+ TrackerClass.noteCommit();
return new DocumentSetAndFlags(rval, rvalBoolean);
@@ -1828,6 +1843,7 @@ public class JobManager implements IJobM
catch (ManifoldCFException e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
{
if (Logging.perf.isDebugEnabled())
@@ -1840,6 +1856,7 @@ public class JobManager implements IJobM
catch (Error e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
throw e;
}
finally
@@ -2442,11 +2459,13 @@ public class JobManager implements IJobM
i++;
}
database.performCommit();
+ TrackerClass.noteCommit();
break;
}
catch (ManifoldCFException e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
{
if (Logging.perf.isDebugEnabled())
@@ -2460,6 +2479,7 @@ public class JobManager implements IJobM
catch (Error e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
throw e;
}
finally
@@ -2607,11 +2627,13 @@ public class JobManager implements IJobM
// we don't delete them here.
database.performCommit();
+ TrackerClass.noteCommit();
return rval;
}
catch (ManifoldCFException e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
{
if (Logging.perf.isDebugEnabled())
@@ -2625,6 +2647,7 @@ public class JobManager implements IJobM
catch (Error e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
throw e;
}
finally
@@ -2987,16 +3010,19 @@ public class JobManager implements IJobM
}
database.performCommit();
+ TrackerClass.noteCommit();
break;
}
catch (Error e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
throw e;
}
catch (ManifoldCFException e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
{
if (Logging.perf.isDebugEnabled())
@@ -3194,11 +3220,13 @@ public class JobManager implements IJobM
}
database.performCommit();
+ TrackerClass.noteCommit();
break;
}
catch (ManifoldCFException e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
{
if (Logging.perf.isDebugEnabled())
@@ -3211,6 +3239,7 @@ public class JobManager implements IJobM
catch (Error e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
throw e;
}
finally
@@ -3289,11 +3318,13 @@ public class JobManager implements IJobM
}
database.performCommit();
+ TrackerClass.noteCommit();
break;
}
catch (ManifoldCFException e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
{
if (Logging.perf.isDebugEnabled())
@@ -3306,6 +3337,7 @@ public class JobManager implements IJobM
catch (Error e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
throw e;
}
finally
@@ -3524,6 +3556,7 @@ public class JobManager implements IJobM
hopCount.recordSeedReferences(jobID,legalLinkTypes,reorderedDocIDHashes,hopcountMethod);
database.performCommit();
+ TrackerClass.noteCommit();
if (Logging.perf.isDebugEnabled())
Logging.perf.debug("Took "+new Long(System.currentTimeMillis()-startTime).toString()+" ms to add "+Integer.toString(reorderedDocIDHashes.length)+
@@ -3544,6 +3577,7 @@ public class JobManager implements IJobM
catch (ManifoldCFException e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
{
if (Logging.perf.isDebugEnabled())
@@ -3557,6 +3591,7 @@ public class JobManager implements IJobM
catch (Error e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
throw e;
}
finally
@@ -4154,6 +4189,7 @@ public class JobManager implements IJobM
jobQueue.reactivateHopcountRemovedRecords(jobID);
database.performCommit();
+ TrackerClass.noteCommit();
if (Logging.perf.isDebugEnabled())
Logging.perf.debug("Took "+new Long(System.currentTimeMillis()-startTime).toString()+" ms to add "+Integer.toString(reorderedDocIDHashes.length)+
@@ -4173,6 +4209,7 @@ public class JobManager implements IJobM
catch (ManifoldCFException e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
{
sleepAmt = getRandomAmount();
@@ -4186,6 +4223,7 @@ public class JobManager implements IJobM
catch (Error e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
throw e;
}
finally
@@ -5377,6 +5415,7 @@ public class JobManager implements IJobM
// No special treatment needed for hopcount or carrydown, since these all get deleted at once
// at the end of the job delete process.
jobQueue.prepareDeleteScan(jobID);
+ TrackerClass.noteCommit();
}
/** Prepare a job to be run.
@@ -5469,11 +5508,13 @@ public class JobManager implements IJobM
jobQueue.prepareFullScan(jobID);
database.performCommit();
+ TrackerClass.noteCommit();
break;
}
catch (ManifoldCFException e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
{
if (Logging.perf.isDebugEnabled())
@@ -5486,6 +5527,7 @@ public class JobManager implements IJobM
catch (Error e)
{
database.signalRollback();
+ TrackerClass.noteRollback();
throw e;
}
finally
Modified: manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java?rev=1464874&r1=1464873&r2=1464874&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java (original)
+++ manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java Fri Apr 5 07:52:13 2013
@@ -390,6 +390,8 @@ public class JobQueue extends org.apache
// Reindex the jobqueue table, since we've probably made lots of bad tuples doing the above operations.
reindexTable();
unconditionallyAnalyzeTables();
+
+ TrackerClass.noteGlobalEvent("Restart");
}
/** Flip all records for a job that have status HOPCOUNTREMOVED back to PENDING.
@@ -407,6 +409,8 @@ public class JobQueue extends org.apache
new UnitaryClause(jobIDField,jobID),
new UnitaryClause(statusField,statusToString(STATUS_HOPCOUNTREMOVED))});
performUpdate(map,"WHERE "+query,list,null);
+
+ TrackerClass.noteJobEvent(jobID,"Map HOPCOUNTREMOVED to PENDING");
}
/** Delete all records for a job that have status HOPCOUNTREMOVED.
@@ -464,6 +468,8 @@ public class JobQueue extends org.apache
statusToString(STATUS_ACTIVEPURGATORY),
statusToString(STATUS_ACTIVENEEDRESCANPURGATORY)})});
performUpdate(map,"WHERE "+query,list,null);
+
+ TrackerClass.noteGlobalEvent("Reset document worker status");
}
/** Reset doc delete worker status.
@@ -479,6 +485,8 @@ public class JobQueue extends org.apache
String query = buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(statusField,statusToString(STATUS_BEINGDELETED))});
performUpdate(map,"WHERE "+query,list,null);
+
+ TrackerClass.noteGlobalEvent("Reset document delete worker status");
}
/** Reset doc cleaning worker status.
@@ -494,6 +502,8 @@ public class JobQueue extends org.apache
String query = buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(statusField,statusToString(STATUS_BEINGCLEANED))});
performUpdate(map,"WHERE "+query,list,null);
+
+ TrackerClass.noteGlobalEvent("Reset document cleanup worker status");
}
/** Prepare for a job delete pass. This will not be called
@@ -545,6 +555,7 @@ public class JobQueue extends org.apache
// Do an analyze, otherwise our plans are going to be crap right off the bat
unconditionallyAnalyzeTables();
+ TrackerClass.noteJobEvent(jobID,"Prepare delete scan");
}
/** Prepare for a "full scan" job. This will not be called
@@ -595,6 +606,8 @@ public class JobQueue extends org.apache
noteModifications(0,2,0);
// Do an analyze, otherwise our plans are going to be crap right off the bat
unconditionallyAnalyzeTables();
+
+ TrackerClass.noteJobEvent(jobID,"Prepare full scan");
}
/** Prepare for a "partial" job. This is called ONLY when the job is inactive.
@@ -654,6 +667,8 @@ public class JobQueue extends org.apache
noteModifications(0,1,0);
// Do an analyze, otherwise our plans are going to be crap right off the bat
unconditionallyAnalyzeTables();
+
+ TrackerClass.noteJobEvent(jobID,"Prepare incremental scan");
}
/** Delete ingested document identifiers (as part of deleting the owning job).
@@ -767,6 +782,7 @@ public class JobQueue extends org.apache
// Leave doc priority unchanged.
break;
default:
+ TrackerClass.printForensics(recID, currentStatus);
throw new ManifoldCFException("Unexpected jobqueue status - record id "+recID.toString()+", expecting active status, saw "+Integer.toString(currentStatus));
}
@@ -779,6 +795,8 @@ public class JobQueue extends org.apache
String query = buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(idField,recID)});
performUpdate(map,"WHERE "+query,list,null);
+
+ TrackerClass.noteRecordEvent(recID, newStatus, "Note completion");
}
/** Either mark a record as hopcountremoved, or set status to "rescan", depending on the
@@ -814,6 +832,7 @@ public class JobQueue extends org.apache
// Leave doc priority unchanged.
break;
default:
+ TrackerClass.printForensics(recID, currentStatus);
throw new ManifoldCFException("Unexpected jobqueue status - record id "+recID.toString()+", expecting active status, saw "+Integer.toString(currentStatus));
}
@@ -826,6 +845,7 @@ public class JobQueue extends org.apache
String query = buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(idField,recID)});
performUpdate(map,"WHERE "+query,list,null);
+ TrackerClass.noteRecordEvent(recID, newStatus, "Update or hopcount remove");
return rval;
}
@@ -846,6 +866,7 @@ public class JobQueue extends org.apache
newStatus = STATUS_ACTIVEPURGATORY;
break;
default:
+ TrackerClass.printForensics(id, currentStatus);
throw new ManifoldCFException("Unexpected status value for jobqueue record "+id.toString()+"; got "+Integer.toString(currentStatus));
}
@@ -856,6 +877,7 @@ public class JobQueue extends org.apache
new UnitaryClause(idField,id)});
performUpdate(map,"WHERE "+query,list,null);
noteModifications(0,1,0);
+ TrackerClass.noteRecordEvent(id, newStatus, "Make active");
}
/** Set the status on a record, including check time and priority.
@@ -887,6 +909,7 @@ public class JobQueue extends org.apache
new UnitaryClause(idField,id)});
performUpdate(map,"WHERE "+query,list,null);
noteModifications(0,1,0);
+ TrackerClass.noteRecordEvent(id, status, "Set status");
}
/** Set the status of a document to "being deleted".
@@ -901,6 +924,7 @@ public class JobQueue extends org.apache
new UnitaryClause(idField,id)});
performUpdate(map,"WHERE "+query,list,null);
noteModifications(0,1,0);
+ TrackerClass.noteRecordEvent(id, STATUS_BEINGDELETED, "Set deleting status");
}
/** Set the status of a document to be "no longer deleting" */
@@ -918,6 +942,7 @@ public class JobQueue extends org.apache
new UnitaryClause(idField,id)});
performUpdate(map,"WHERE "+query,list,null);
noteModifications(0,1,0);
+ TrackerClass.noteRecordEvent(id, STATUS_ELIGIBLEFORDELETE, "Set undeleting status");
}
/** Set the status of a document to "being cleaned".
@@ -932,6 +957,7 @@ public class JobQueue extends org.apache
new UnitaryClause(idField,id)});
performUpdate(map,"WHERE "+query,list,null);
noteModifications(0,1,0);
+ TrackerClass.noteRecordEvent(id, STATUS_BEINGCLEANED, "Set cleaning status");
}
/** Set the status of a document to be "no longer cleaning" */
@@ -949,6 +975,7 @@ public class JobQueue extends org.apache
new UnitaryClause(idField,id)});
performUpdate(map,"WHERE "+query,list,null);
noteModifications(0,1,0);
+ TrackerClass.noteRecordEvent(id, STATUS_PURGATORY, "Set uncleaning status");
}
/** Remove multiple records entirely.
@@ -1040,6 +1067,7 @@ public class JobQueue extends org.apache
case STATUS_PURGATORY:
// Set the status and time both
map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
+ TrackerClass.noteRecordEvent(recordID, STATUS_PENDINGPURGATORY, "Update existing record initial");
if (desiredExecuteTime == -1L)
map.put(checkTimeField,new Long(0L));
else
@@ -1130,6 +1158,7 @@ public class JobQueue extends org.apache
performInsert(map,null);
prereqEventManager.addRows(recordID,prereqEvents);
noteModifications(1,0,0);
+ TrackerClass.noteRecordEvent(recordID, STATUS_PENDING, "Create initial");
}
/** Note the remaining documents that do NOT need to be queued. These are noted so that the
@@ -1317,6 +1346,7 @@ public class JobQueue extends org.apache
case STATUS_UNCHANGED:
// Set the status and time both
map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
+ TrackerClass.noteRecordEvent(recordID, STATUS_PENDINGPURGATORY, "Update existing");
map.put(checkTimeField,new Long(desiredExecuteTime));
map.put(checkActionField,actionToString(ACTION_RESCAN));
map.put(failTimeField,null);
@@ -1334,6 +1364,7 @@ public class JobQueue extends org.apache
// The document has been processed before, so it has to go into PENDINGPURGATORY.
// Set the status and time both
map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
+ TrackerClass.noteRecordEvent(recordID, STATUS_PENDINGPURGATORY, "Update existing");
map.put(checkTimeField,new Long(desiredExecuteTime));
map.put(checkActionField,actionToString(ACTION_RESCAN));
map.put(failTimeField,null);
@@ -1363,6 +1394,7 @@ public class JobQueue extends org.apache
// Flip the state to the new one, and set the document priority at this time too - it will be preserved when the
// processing is completed.
map.put(statusField,statusToString(STATUS_ACTIVENEEDRESCAN));
+ TrackerClass.noteRecordEvent(recordID, STATUS_ACTIVENEEDRESCAN, "Update existing");
map.put(checkTimeField,new Long(desiredExecuteTime));
map.put(checkActionField,actionToString(ACTION_RESCAN));
map.put(failTimeField,null);
@@ -1387,6 +1419,7 @@ public class JobQueue extends org.apache
// Flip the state to the new one, and set the document priority at this time too - it will be preserved when the
// processing is completed.
map.put(statusField,statusToString(STATUS_ACTIVENEEDRESCANPURGATORY));
+ TrackerClass.noteRecordEvent(recordID, STATUS_ACTIVENEEDRESCANPURGATORY, "Update existing");
map.put(checkTimeField,new Long(desiredExecuteTime));
map.put(checkActionField,actionToString(ACTION_RESCAN));
map.put(failTimeField,null);
@@ -1454,6 +1487,8 @@ public class JobQueue extends org.apache
performInsert(map,null);
prereqEventManager.addRows(recordID,prereqEvents);
noteModifications(1,0,0);
+ TrackerClass.noteRecordEvent(recordID, STATUS_PENDING, "Create new");
+
}
// Methods to convert status strings to integers and back