You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/04/07 21:48:10 UTC
svn commit: r1465451 - in /manifoldcf/branches/CONNECTORS-590-2/framework:
core/src/main/java/org/apache/manifoldcf/core/system/
core/src/test/java/org/apache/manifoldcf/core/tests/
pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ pull-agen...
Author: kwright
Date: Sun Apr 7 19:48:09 2013
New Revision: 1465451
URL: http://svn.apache.org/r1465451
Log:
Revise and update diagnostic logging so that it will be unobtrusive unless called for.
Modified:
manifoldcf/branches/CONNECTORS-590-2/framework/core/src/main/java/org/apache/manifoldcf/core/system/Logging.java
manifoldcf/branches/CONNECTORS-590-2/framework/core/src/test/java/org/apache/manifoldcf/core/tests/Base.java
manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/TrackerClass.java
manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
Modified: manifoldcf/branches/CONNECTORS-590-2/framework/core/src/main/java/org/apache/manifoldcf/core/system/Logging.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-590-2/framework/core/src/main/java/org/apache/manifoldcf/core/system/Logging.java?rev=1465451&r1=1465450&r2=1465451&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-590-2/framework/core/src/main/java/org/apache/manifoldcf/core/system/Logging.java (original)
+++ manifoldcf/branches/CONNECTORS-590-2/framework/core/src/main/java/org/apache/manifoldcf/core/system/Logging.java Sun Apr 7 19:48:09 2013
@@ -45,6 +45,7 @@ public class Logging
public static Logger cache = null;
public static Logger keystore = null;
public static Logger perf = null;
+ public static Logger diagnostics = null;
private static HashMap loggerTable = null;
private static HashMap logLevelMap = null;
@@ -90,6 +91,7 @@ public class Logging
cache = newLogger("org.apache.manifoldcf.cache");
keystore = newLogger("org.apache.manifoldcf.keystore");
perf = newLogger("org.apache.manifoldcf.perf");
+ diagnostics = newLogger("org.apache.manifoldcf.diagnostics");
}
/** Reset all loggers
Modified: manifoldcf/branches/CONNECTORS-590-2/framework/core/src/test/java/org/apache/manifoldcf/core/tests/Base.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-590-2/framework/core/src/test/java/org/apache/manifoldcf/core/tests/Base.java?rev=1465451&r1=1465450&r2=1465451&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-590-2/framework/core/src/test/java/org/apache/manifoldcf/core/tests/Base.java (original)
+++ manifoldcf/branches/CONNECTORS-590-2/framework/core/src/test/java/org/apache/manifoldcf/core/tests/Base.java Sun Apr 7 19:48:09 2013
@@ -122,7 +122,8 @@ public class Base
{
output.append(
" <property name=\"org.apache.manifoldcf.logconfigfile\" value=\""+loggingFile.getAbsolutePath().replaceAll("\\\\","/")+"\"/>\n"+
- " <property name=\"org.apache.manifoldcf.connectorsconfigurationfile\" value=\""+connectorFile.getAbsolutePath().replaceAll("\\\\","/")+"\"/>\n"
+ " <property name=\"org.apache.manifoldcf.connectorsconfigurationfile\" value=\""+connectorFile.getAbsolutePath().replaceAll("\\\\","/")+"\"/>\n"+
+ " <property name=\"org.apache.manifoldcf.diagnostics\" value=\"DEBUG\"/>\n"
);
}
Modified: manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1465451&r1=1465450&r2=1465451&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java (original)
+++ manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java Sun Apr 7 19:48:09 2013
@@ -644,6 +644,7 @@ public class JobManager implements IJobM
hopCount.reset();
// Clean up carrydown stuff
carryDown.reset();
+ TrackerClass.notePrecommit();
database.performCommit();
TrackerClass.noteCommit();
Logging.jobs.debug("Reset complete");
@@ -689,6 +690,7 @@ public class JobManager implements IJobM
try
{
jobQueue.resetDocumentWorkerStatus();
+ TrackerClass.notePrecommit();
database.performCommit();
TrackerClass.noteCommit();
break;
@@ -737,6 +739,7 @@ public class JobManager implements IJobM
throws ManifoldCFException
{
Logging.jobs.debug("Resetting doc deleting status");
+ TrackerClass.notePrecommit();
jobQueue.resetDocDeleteWorkerStatus();
TrackerClass.noteCommit();
Logging.jobs.debug("Reset complete");
@@ -748,6 +751,7 @@ public class JobManager implements IJobM
throws ManifoldCFException
{
Logging.jobs.debug("Resetting doc cleaning status");
+ TrackerClass.notePrecommit();
jobQueue.resetDocCleanupWorkerStatus();
TrackerClass.noteCommit();
Logging.jobs.debug("Reset complete");
@@ -1003,7 +1007,8 @@ public class JobManager implements IJobM
rval[i++] = dd;
jobQueue.setCleaningStatus(dd.getID());
}
-
+
+ TrackerClass.notePrecommit();
database.performCommit();
TrackerClass.noteCommit();
@@ -1268,6 +1273,7 @@ public class JobManager implements IJobM
i++;
}
+ TrackerClass.notePrecommit();
database.performCommit();
TrackerClass.noteCommit();
@@ -1834,6 +1840,7 @@ public class JobManager implements IJobM
jobQueue.updateActiveRecord(dd.getID(),((Integer)statusMap.get(compositeDocID)).intValue());
}
+ TrackerClass.notePrecommit();
database.performCommit();
TrackerClass.noteCommit();
@@ -2342,7 +2349,9 @@ public class JobManager implements IJobM
i++;
}
+ TrackerClass.notePrecommit();
database.performCommit();
+ TrackerClass.noteCommit();
break;
}
catch (ManifoldCFException e)
@@ -2446,8 +2455,10 @@ public class JobManager implements IJobM
ArrayList list = new ArrayList();
String query = database.buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(jobQueue.idField,dd.getID())});
+ TrackerClass.notePreread(dd.getID());
IResultSet set = database.performQuery("SELECT "+jobQueue.statusField+" FROM "+jobQueue.getTableName()+" WHERE "+
query+" FOR UPDATE",list,null,null);
+ TrackerClass.noteRead(dd.getID());
if (set.getRowCount() > 0)
{
IResultRow row = set.getRow(0);
@@ -2458,6 +2469,7 @@ public class JobManager implements IJobM
}
i++;
}
+ TrackerClass.notePrecommit();
database.performCommit();
TrackerClass.noteCommit();
break;
@@ -2597,8 +2609,10 @@ public class JobManager implements IJobM
ArrayList list = new ArrayList();
String query = database.buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(jobQueue.idField,dd.getID())});
+ TrackerClass.notePreread(dd.getID());
IResultSet set = database.performQuery("SELECT "+jobQueue.statusField+" FROM "+jobQueue.getTableName()+" WHERE "+
query+" FOR UPDATE",list,null,null);
+ TrackerClass.noteRead(dd.getID());
if (set.getRowCount() > 0)
{
IResultRow row = set.getRow(0);
@@ -2626,6 +2640,7 @@ public class JobManager implements IJobM
// Since hopcount inheritance and prerequisites came from the addDocument() method,
// we don't delete them here.
+ TrackerClass.notePrecommit();
database.performCommit();
TrackerClass.noteCommit();
return rval;
@@ -3009,6 +3024,7 @@ public class JobManager implements IJobM
i++;
}
+ TrackerClass.notePrecommit();
database.performCommit();
TrackerClass.noteCommit();
break;
@@ -3219,6 +3235,7 @@ public class JobManager implements IJobM
i++;
}
+ TrackerClass.notePrecommit();
database.performCommit();
TrackerClass.noteCommit();
break;
@@ -3317,6 +3334,7 @@ public class JobManager implements IJobM
i++;
}
+ TrackerClass.notePrecommit();
database.performCommit();
TrackerClass.noteCommit();
break;
@@ -3555,6 +3573,7 @@ public class JobManager implements IJobM
if (legalLinkTypes.length > 0)
hopCount.recordSeedReferences(jobID,legalLinkTypes,reorderedDocIDHashes,hopcountMethod);
+ TrackerClass.notePrecommit();
database.performCommit();
TrackerClass.noteCommit();
@@ -4188,6 +4207,7 @@ public class JobManager implements IJobM
if (reactivateRemovedHopcountRecords)
jobQueue.reactivateHopcountRemovedRecords(jobID);
+ TrackerClass.notePrecommit();
database.performCommit();
TrackerClass.noteCommit();
@@ -5414,6 +5434,7 @@ public class JobManager implements IJobM
{
// No special treatment needed for hopcount or carrydown, since these all get deleted at once
// at the end of the job delete process.
+ TrackerClass.notePrecommit();
jobQueue.prepareDeleteScan(jobID);
TrackerClass.noteCommit();
}
@@ -5507,6 +5528,7 @@ public class JobManager implements IJobM
}
jobQueue.prepareFullScan(jobID);
+ TrackerClass.notePrecommit();
database.performCommit();
TrackerClass.noteCommit();
break;
Modified: manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java?rev=1465451&r1=1465450&r2=1465451&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java (original)
+++ manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java Sun Apr 7 19:48:09 2013
@@ -391,7 +391,7 @@ public class JobQueue extends org.apache
reindexTable();
unconditionallyAnalyzeTables();
- TrackerClass.noteGlobalEvent("Restart");
+ TrackerClass.noteGlobalChange("Restart");
}
/** Flip all records for a job that have status HOPCOUNTREMOVED back to PENDING.
@@ -410,7 +410,7 @@ public class JobQueue extends org.apache
new UnitaryClause(statusField,statusToString(STATUS_HOPCOUNTREMOVED))});
performUpdate(map,"WHERE "+query,list,null);
- TrackerClass.noteJobEvent(jobID,"Map HOPCOUNTREMOVED to PENDING");
+ TrackerClass.noteJobChange(jobID,"Map HOPCOUNTREMOVED to PENDING");
}
/** Delete all records for a job that have status HOPCOUNTREMOVED.
@@ -469,7 +469,7 @@ public class JobQueue extends org.apache
statusToString(STATUS_ACTIVENEEDRESCANPURGATORY)})});
performUpdate(map,"WHERE "+query,list,null);
- TrackerClass.noteGlobalEvent("Reset document worker status");
+ TrackerClass.noteGlobalChange("Reset document worker status");
}
/** Reset doc delete worker status.
@@ -486,7 +486,7 @@ public class JobQueue extends org.apache
new UnitaryClause(statusField,statusToString(STATUS_BEINGDELETED))});
performUpdate(map,"WHERE "+query,list,null);
- TrackerClass.noteGlobalEvent("Reset document delete worker status");
+ TrackerClass.noteGlobalChange("Reset document delete worker status");
}
/** Reset doc cleaning worker status.
@@ -503,7 +503,7 @@ public class JobQueue extends org.apache
new UnitaryClause(statusField,statusToString(STATUS_BEINGCLEANED))});
performUpdate(map,"WHERE "+query,list,null);
- TrackerClass.noteGlobalEvent("Reset document cleanup worker status");
+ TrackerClass.noteGlobalChange("Reset document cleanup worker status");
}
/** Prepare for a job delete pass. This will not be called
@@ -555,7 +555,7 @@ public class JobQueue extends org.apache
// Do an analyze, otherwise our plans are going to be crap right off the bat
unconditionallyAnalyzeTables();
- TrackerClass.noteJobEvent(jobID,"Prepare delete scan");
+ TrackerClass.noteJobChange(jobID,"Prepare delete scan");
}
/** Prepare for a "full scan" job. This will not be called
@@ -607,7 +607,7 @@ public class JobQueue extends org.apache
// Do an analyze, otherwise our plans are going to be crap right off the bat
unconditionallyAnalyzeTables();
- TrackerClass.noteJobEvent(jobID,"Prepare full scan");
+ TrackerClass.noteJobChange(jobID,"Prepare full scan");
}
/** Prepare for a "partial" job. This is called ONLY when the job is inactive.
@@ -668,7 +668,7 @@ public class JobQueue extends org.apache
// Do an analyze, otherwise our plans are going to be crap right off the bat
unconditionallyAnalyzeTables();
- TrackerClass.noteJobEvent(jobID,"Prepare incremental scan");
+ TrackerClass.noteJobChange(jobID,"Prepare incremental scan");
}
/** Delete ingested document identifiers (as part of deleting the owning job).
@@ -796,7 +796,7 @@ public class JobQueue extends org.apache
new UnitaryClause(idField,recID)});
performUpdate(map,"WHERE "+query,list,null);
- TrackerClass.noteRecordEvent(recID, newStatus, "Note completion");
+ TrackerClass.noteRecordChange(recID, newStatus, "Note completion");
}
/** Either mark a record as hopcountremoved, or set status to "rescan", depending on the
@@ -845,7 +845,7 @@ public class JobQueue extends org.apache
String query = buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(idField,recID)});
performUpdate(map,"WHERE "+query,list,null);
- TrackerClass.noteRecordEvent(recID, newStatus, "Update or hopcount remove");
+ TrackerClass.noteRecordChange(recID, newStatus, "Update or hopcount remove");
return rval;
}
@@ -877,7 +877,7 @@ public class JobQueue extends org.apache
new UnitaryClause(idField,id)});
performUpdate(map,"WHERE "+query,list,null);
noteModifications(0,1,0);
- TrackerClass.noteRecordEvent(id, newStatus, "Make active");
+ TrackerClass.noteRecordChange(id, newStatus, "Make active");
}
/** Set the status on a record, including check time and priority.
@@ -909,7 +909,7 @@ public class JobQueue extends org.apache
new UnitaryClause(idField,id)});
performUpdate(map,"WHERE "+query,list,null);
noteModifications(0,1,0);
- TrackerClass.noteRecordEvent(id, status, "Set status");
+ TrackerClass.noteRecordChange(id, status, "Set status");
}
/** Set the status of a document to "being deleted".
@@ -924,7 +924,7 @@ public class JobQueue extends org.apache
new UnitaryClause(idField,id)});
performUpdate(map,"WHERE "+query,list,null);
noteModifications(0,1,0);
- TrackerClass.noteRecordEvent(id, STATUS_BEINGDELETED, "Set deleting status");
+ TrackerClass.noteRecordChange(id, STATUS_BEINGDELETED, "Set deleting status");
}
/** Set the status of a document to be "no longer deleting" */
@@ -942,7 +942,7 @@ public class JobQueue extends org.apache
new UnitaryClause(idField,id)});
performUpdate(map,"WHERE "+query,list,null);
noteModifications(0,1,0);
- TrackerClass.noteRecordEvent(id, STATUS_ELIGIBLEFORDELETE, "Set undeleting status");
+ TrackerClass.noteRecordChange(id, STATUS_ELIGIBLEFORDELETE, "Set undeleting status");
}
/** Set the status of a document to "being cleaned".
@@ -957,7 +957,7 @@ public class JobQueue extends org.apache
new UnitaryClause(idField,id)});
performUpdate(map,"WHERE "+query,list,null);
noteModifications(0,1,0);
- TrackerClass.noteRecordEvent(id, STATUS_BEINGCLEANED, "Set cleaning status");
+ TrackerClass.noteRecordChange(id, STATUS_BEINGCLEANED, "Set cleaning status");
}
/** Set the status of a document to be "no longer cleaning" */
@@ -975,7 +975,7 @@ public class JobQueue extends org.apache
new UnitaryClause(idField,id)});
performUpdate(map,"WHERE "+query,list,null);
noteModifications(0,1,0);
- TrackerClass.noteRecordEvent(id, STATUS_PURGATORY, "Set uncleaning status");
+ TrackerClass.noteRecordChange(id, STATUS_PURGATORY, "Set uncleaning status");
}
/** Remove multiple records entirely.
@@ -1067,7 +1067,7 @@ public class JobQueue extends org.apache
case STATUS_PURGATORY:
// Set the status and time both
map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
- TrackerClass.noteRecordEvent(recordID, STATUS_PENDINGPURGATORY, "Update existing record initial");
+ TrackerClass.noteRecordChange(recordID, STATUS_PENDINGPURGATORY, "Update existing record initial");
if (desiredExecuteTime == -1L)
map.put(checkTimeField,new Long(0L));
else
@@ -1158,7 +1158,7 @@ public class JobQueue extends org.apache
performInsert(map,null);
prereqEventManager.addRows(recordID,prereqEvents);
noteModifications(1,0,0);
- TrackerClass.noteRecordEvent(recordID, STATUS_PENDING, "Create initial");
+ TrackerClass.noteRecordChange(recordID, STATUS_PENDING, "Create initial");
}
/** Note the remaining documents that do NOT need to be queued. These are noted so that the
@@ -1346,7 +1346,7 @@ public class JobQueue extends org.apache
case STATUS_UNCHANGED:
// Set the status and time both
map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
- TrackerClass.noteRecordEvent(recordID, STATUS_PENDINGPURGATORY, "Update existing");
+ TrackerClass.noteRecordChange(recordID, STATUS_PENDINGPURGATORY, "Update existing");
map.put(checkTimeField,new Long(desiredExecuteTime));
map.put(checkActionField,actionToString(ACTION_RESCAN));
map.put(failTimeField,null);
@@ -1364,7 +1364,7 @@ public class JobQueue extends org.apache
// The document has been processed before, so it has to go into PENDINGPURGATORY.
// Set the status and time both
map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
- TrackerClass.noteRecordEvent(recordID, STATUS_PENDINGPURGATORY, "Update existing");
+ TrackerClass.noteRecordChange(recordID, STATUS_PENDINGPURGATORY, "Update existing");
map.put(checkTimeField,new Long(desiredExecuteTime));
map.put(checkActionField,actionToString(ACTION_RESCAN));
map.put(failTimeField,null);
@@ -1394,7 +1394,7 @@ public class JobQueue extends org.apache
// Flip the state to the new one, and set the document priority at this time too - it will be preserved when the
// processing is completed.
map.put(statusField,statusToString(STATUS_ACTIVENEEDRESCAN));
- TrackerClass.noteRecordEvent(recordID, STATUS_ACTIVENEEDRESCAN, "Update existing");
+ TrackerClass.noteRecordChange(recordID, STATUS_ACTIVENEEDRESCAN, "Update existing");
map.put(checkTimeField,new Long(desiredExecuteTime));
map.put(checkActionField,actionToString(ACTION_RESCAN));
map.put(failTimeField,null);
@@ -1419,7 +1419,7 @@ public class JobQueue extends org.apache
// Flip the state to the new one, and set the document priority at this time too - it will be preserved when the
// processing is completed.
map.put(statusField,statusToString(STATUS_ACTIVENEEDRESCANPURGATORY));
- TrackerClass.noteRecordEvent(recordID, STATUS_ACTIVENEEDRESCANPURGATORY, "Update existing");
+ TrackerClass.noteRecordChange(recordID, STATUS_ACTIVENEEDRESCANPURGATORY, "Update existing");
map.put(checkTimeField,new Long(desiredExecuteTime));
map.put(checkActionField,actionToString(ACTION_RESCAN));
map.put(failTimeField,null);
@@ -1487,7 +1487,7 @@ public class JobQueue extends org.apache
performInsert(map,null);
prereqEventManager.addRows(recordID,prereqEvents);
noteModifications(1,0,0);
- TrackerClass.noteRecordEvent(recordID, STATUS_PENDING, "Create new");
+ TrackerClass.noteRecordChange(recordID, STATUS_PENDING, "Create new");
}
Modified: manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/TrackerClass.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/TrackerClass.java?rev=1465451&r1=1465450&r2=1465451&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/TrackerClass.java (original)
+++ manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/TrackerClass.java Sun Apr 7 19:48:09 2013
@@ -22,6 +22,8 @@ package org.apache.manifoldcf.crawler.jo
import java.util.*;
import java.io.*;
+import org.apache.manifoldcf.crawler.system.Logging;
+
/** Debugging class to keep track of recent modifications to the jobqueue table,
* along with context as to where it occurred. If a jobqueue state error occurs,
* we can then print out all of the pertinent history and find the culprit.
@@ -36,7 +38,7 @@ public class TrackerClass
protected final static Map<String,TransactionData> transactionData = new HashMap<String,TransactionData>();
// Modification history
- protected final static List<TransactionData> history = new ArrayList<TransactionData>();
+ protected final static List<HistoryRecord> history = new ArrayList<HistoryRecord>();
// Place where we keep track of individual modifications
private TrackerClass()
@@ -44,28 +46,31 @@ public class TrackerClass
}
/** Add a single record event, as yet uncommitted */
- public static void noteRecordEvent(Long recordID, int newStatus, String description)
+ public static void noteRecordChange(Long recordID, int newStatus, String description)
{
- addEvent(new RecordEvent(recordID, newStatus, new Exception(description)));
+ if (Logging.diagnostics.isDebugEnabled())
+ addChange(new RecordChange(recordID, newStatus, description));
}
/** Add a global event, as yet uncommitted, which has the potential
* to affect any record's state in a given job.
*/
- public static void noteJobEvent(Long jobID, String description)
+ public static void noteJobChange(Long jobID, String description)
{
- addEvent(new JobEvent(jobID, new Exception(description)));
+ if (Logging.diagnostics.isDebugEnabled())
+ addChange(new JobChange(jobID, description));
}
/** Add a global event, as yet uncommitted, which has the potential
* to affect the state of any record.
*/
- public static void noteGlobalEvent(String description)
+ public static void noteGlobalChange(String description)
{
- addEvent(new GlobalEvent(new Exception(description)));
+ if (Logging.diagnostics.isDebugEnabled())
+ addChange(new GlobalChange(description));
}
- protected static void addEvent(HistoryRecord hr)
+ protected static void addChange(DataChange dc)
{
String threadName = Thread.currentThread().getName();
TransactionData td;
@@ -78,33 +83,105 @@ public class TrackerClass
transactionData.put(threadName,td);
}
}
- td.addEvent(hr);
+ td.addChange(dc);
+ }
+
+ /** Note that we are about to commit.
+ */
+ public static void notePrecommit()
+ {
+ if (!Logging.diagnostics.isDebugEnabled())
+ return;
+
+ long currentTime = System.currentTimeMillis();
+ String threadName = Thread.currentThread().getName();
+
+ TransactionData td;
+ synchronized (transactionData)
+ {
+ td = transactionData.get(threadName);
+ }
+
+ if (td == null)
+ return;
+
+ HistoryRecord hr = new PrecommitEvent(new Exception("Precommit stack trace"),currentTime,threadName,td);
+
+ synchronized (history)
+ {
+ history.add(hr);
+ }
+ }
+
+ /** Note a read status operation.
+ */
+ public static void noteRead(Long recordID)
+ {
+ if (!Logging.diagnostics.isDebugEnabled())
+ return;
+
+ long currentTime = System.currentTimeMillis();
+ String threadName = Thread.currentThread().getName();
+
+ HistoryRecord hr = new ReadEvent(new Exception("Read stack trace"),currentTime,threadName,recordID);
+
+ synchronized (history)
+ {
+ history.add(hr);
+ }
+ }
+
+ /** Note about to read status operation.
+ */
+ public static void notePreread(Long recordID)
+ {
+ if (!Logging.diagnostics.isDebugEnabled())
+ return;
+
+ long currentTime = System.currentTimeMillis();
+ String threadName = Thread.currentThread().getName();
+
+ HistoryRecord hr = new PrereadEvent(new Exception("Pre-read stack trace"),currentTime,threadName,recordID);
+
+ synchronized (history)
+ {
+ history.add(hr);
+ }
}
/** Note a commit operation.
*/
public static void noteCommit()
{
+ if (!Logging.diagnostics.isDebugEnabled())
+ return;
+
long currentTime = System.currentTimeMillis();
String threadName = Thread.currentThread().getName();
+
TransactionData td;
synchronized (transactionData)
{
td = transactionData.get(threadName);
transactionData.remove(threadName);
}
+
if (td == null)
return;
+
+ HistoryRecord hr = new CommitEvent(new Exception("Commit stack trace"),currentTime,threadName,td);
+
// Only keep stuff around for an hour
long removalCutoff = currentTime - HISTORY_LENGTH;
synchronized (history)
{
- history.add(td);
+ history.add(hr);
// Clean out older records
+ // MHL - this logic is wrong
while (history.size() > 0)
{
- TransactionData td2 = history.get(0);
- if (td2.isFlushable(removalCutoff))
+ HistoryRecord oldRecord = history.get(0);
+ if (oldRecord.isFlushable(removalCutoff))
history.remove(0);
else
break;
@@ -117,6 +194,9 @@ public class TrackerClass
*/
public static void noteRollback()
{
+ if (!Logging.diagnostics.isDebugEnabled())
+ return;
+
String threadName = Thread.currentThread().getName();
synchronized (transactionData)
{
@@ -126,31 +206,27 @@ public class TrackerClass
public static void printForensics(Long recordID, int existingStatus)
{
- synchronized (transactionData)
+ if (Logging.diagnostics.isDebugEnabled())
{
- synchronized (history)
+ synchronized (transactionData)
{
- System.err.println("---- Forensics for record "+recordID+", current status: "+existingStatus+" ----");
- System.err.println("--Current stack trace--");
- StringWriter sw = new StringWriter();
- new Exception("Unexpected jobqueue status").printStackTrace(new PrintWriter(sw,true));
- System.err.print(sw.toString());
- System.err.println("--Active transactions--");
- for (String threadName : transactionData.keySet())
+ synchronized (history)
{
- for (HistoryRecord hr : transactionData.get(threadName).getEvents())
+ Logging.diagnostics.debug("==== Forensics for record "+recordID+", current status: "+existingStatus+" ====");
+ Logging.diagnostics.debug("=== Current stack trace ===",new Exception("Forensics stack trace"));
+ Logging.diagnostics.debug("=== Active transactions ===");
+ for (String threadName : transactionData.keySet())
{
- if (hr.applies(recordID))
+ for (DataChange dc : transactionData.get(threadName).getChanges())
{
- System.err.println("Thread '"+threadName+"' was active:");
- hr.print();
+ if (dc.applies(recordID))
+ {
+ Logging.diagnostics.debug("Thread '"+threadName+"' was doing things to this record: " + dc.getDescription());
+ }
}
}
- }
- System.err.println("--Pertinent History--");
- for (TransactionData td : history)
- {
- for (HistoryRecord hr : td.getEvents())
+ Logging.diagnostics.debug("=== Pertinent History ===");
+ for (HistoryRecord hr : history)
{
if (hr.applies(recordID))
{
@@ -160,100 +236,226 @@ public class TrackerClass
}
}
}
-
}
protected static class TransactionData
{
- protected final List<HistoryRecord> transactionEvents = new ArrayList<HistoryRecord>();
- protected long timestamp;
+ protected final List<DataChange> changes = new ArrayList<DataChange>();
public TransactionData()
{
- timestamp = System.currentTimeMillis();
}
- public void addEvent(HistoryRecord event)
+ public void addChange(DataChange change)
{
- transactionEvents.add(event);
+ changes.add(change);
}
- public List<HistoryRecord> getEvents()
+ public List<DataChange> getChanges()
{
- return transactionEvents;
+ return changes;
}
- public boolean isFlushable(long cutoffTime)
+ public boolean applies(Long recordID)
{
- return cutoffTime > timestamp;
+ for (DataChange dc : changes)
+ {
+ if (dc.applies(recordID))
+ return true;
+ }
+ return false;
}
}
+ protected abstract static class DataChange
+ {
+ protected final String description;
+
+ public DataChange(String description)
+ {
+ this.description = description;
+ }
+
+ public String getDescription()
+ {
+ return description;
+ }
+
+ public abstract boolean applies(Long recordID);
+
+ }
+
protected abstract static class HistoryRecord
{
- protected long timestamp;
- protected Exception trace;
+ protected final long timestamp;
+ protected final Exception trace;
+ protected final String threadName;
- public HistoryRecord(Exception trace)
+ public HistoryRecord(Exception trace, long timestamp, String threadName)
{
this.trace = trace;
- this.timestamp = System.currentTimeMillis();
+ this.timestamp = timestamp;
+ this.threadName = threadName;
}
- public void print()
+ public void print(String description)
{
- System.err.println(" at "+new Long(timestamp)+", location: ");
- StringWriter sw = new StringWriter();
- trace.printStackTrace(new PrintWriter(sw,true));
- System.err.print(sw.toString());
+ Logging.diagnostics.debug("== "+description+" by '"+threadName+"' at "+new Long(timestamp)+" ==",trace);
}
+ public boolean isFlushable(long timestamp)
+ {
+ return this.timestamp < timestamp;
+ }
+
public abstract boolean applies(Long recordID);
+ public abstract void print();
+
+ }
+
+ protected static class CommitEvent extends HistoryRecord
+ {
+ protected final TransactionData transactionData;
+
+ public CommitEvent(Exception trace, long timestamp, String threadName, TransactionData transactionData)
+ {
+ super(trace,timestamp,threadName);
+ this.transactionData = transactionData;
+ }
+
+ @Override
+ public void print()
+ {
+ super.print("Commit transaction");
+ Logging.diagnostics.debug(" Transaction includes:");
+ for (DataChange dc : transactionData.getChanges())
+ {
+ Logging.diagnostics.debug(" "+dc.getDescription());
+ }
+ }
+
+ @Override
+ public boolean applies(Long recordID)
+ {
+ return transactionData.applies(recordID);
+ }
+
+ }
+
+ protected static class PrecommitEvent extends HistoryRecord
+ {
+ protected final TransactionData transactionData;
+
+ public PrecommitEvent(Exception trace, long timestamp, String threadName, TransactionData transactionData)
+ {
+ super(trace,timestamp,threadName);
+ this.transactionData = transactionData;
+ }
+
+ @Override
+ public void print()
+ {
+ super.print("About to commit transaction");
+ Logging.diagnostics.debug(" Transaction includes:");
+ for (DataChange dc : transactionData.getChanges())
+ {
+ Logging.diagnostics.debug(" "+dc.getDescription());
+ }
+ }
+
+ @Override
+ public boolean applies(Long recordID)
+ {
+ return transactionData.applies(recordID);
+ }
}
- protected static class RecordEvent extends HistoryRecord
+ protected static class ReadEvent extends HistoryRecord
{
- protected Long recordID;
- protected int newStatus;
+ protected final Long recordID;
- public RecordEvent(Long recordID, int newStatus, Exception trace)
+ public ReadEvent(Exception trace, long timestamp, String threadName, Long recordID)
{
- super(trace);
+ super(trace,timestamp,threadName);
this.recordID = recordID;
- this.newStatus = newStatus;
}
@Override
public void print()
{
- System.err.println("Record "+recordID+" status modified to "+newStatus);
- super.print();
+ super.print("Read status");
}
@Override
public boolean applies(Long recordID)
{
- return this.recordID.equals(recordID);
+ return recordID.equals(this.recordID);
}
+ }
+ protected static class PrereadEvent extends HistoryRecord
+ {
+ protected final Long recordID;
+
+ public PrereadEvent(Exception trace, long timestamp, String threadName, Long recordID)
+ {
+ super(trace,timestamp,threadName);
+ this.recordID = recordID;
+ }
+
+ @Override
+ public void print()
+ {
+ super.print("About to read status");
+ }
+
+ @Override
+ public boolean applies(Long recordID)
+ {
+ return recordID.equals(this.recordID);
+ }
}
- protected static class JobEvent extends HistoryRecord
+ protected static class RecordChange extends DataChange
{
- protected Long jobID;
+ protected final Long recordID;
+ protected final int newStatus;
- public JobEvent(Long jobID, Exception trace)
+ public RecordChange(Long recordID, int newStatus, String description)
{
- super(trace);
+ super(description);
+ this.recordID = recordID;
+ this.newStatus = newStatus;
+ }
+
+ @Override
+ public String getDescription()
+ {
+ return "Record "+recordID+" status modified to "+newStatus+": "+super.getDescription();
+ }
+
+ @Override
+ public boolean applies(Long recordID)
+ {
+ return recordID.equals(this.recordID);
+ }
+ }
+
+ protected static class JobChange extends DataChange
+ {
+ protected final Long jobID;
+
+ public JobChange(Long jobID, String description)
+ {
+ super(description);
this.jobID = jobID;
}
@Override
- public void print()
+ public String getDescription()
{
- System.err.println("All job related records modified for job "+jobID);
- super.print();
+ return "All job related records modified for job "+jobID+": "+super.getDescription();
}
@Override
@@ -263,18 +465,17 @@ public class TrackerClass
}
}
- protected static class GlobalEvent extends HistoryRecord
+ protected static class GlobalChange extends DataChange
{
- public GlobalEvent(Exception trace)
+ public GlobalChange(String description)
{
- super(trace);
+ super(description);
}
@Override
- public void print()
+ public String getDescription()
{
- System.err.println("All records modified");
- super.print();
+ return "All records modified: "+super.getDescription();
}
@Override
Modified: manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1465451&r1=1465450&r2=1465451&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java (original)
+++ manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java Sun Apr 7 19:48:09 2013
@@ -647,11 +647,14 @@ public class ManifoldCF extends org.apac
protected static class InitializationThread extends Thread
{
- protected QueueTracker queueTracker;
+ protected final QueueTracker queueTracker;
public InitializationThread(QueueTracker queueTracker)
{
+ super();
this.queueTracker = queueTracker;
+ setName("Initialization thread");
+ setDaemon(true);
}
public void run()