You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/03/16 22:03:09 UTC
svn commit: r1457305 [2/3] - in /manifoldcf/trunk: ./
connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/
framework/crawler-ui/src/main/webapp/
framework/pull-agent/src/main/java/org/apache/manifoldcf/craw...
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java Sat Mar 16 21:03:08 2013
@@ -98,7 +98,7 @@ public class JobManager implements IJobM
throws java.io.IOException, ManifoldCFException
{
// Write a version indicator
- ManifoldCF.writeDword(os,2);
+ ManifoldCF.writeDword(os,3);
// Get the job list
IJobDescription[] list = getAllJobs();
// Write the number of authorities
@@ -136,6 +136,7 @@ public class JobManager implements IJobM
writeEnumeratedValues(os,sr.getMinutesOfHour());
ManifoldCF.writeString(os,sr.getTimezone());
ManifoldCF.writeLong(os,sr.getDuration());
+ ManifoldCF.writeByte(os,sr.getRequestMinimum()?1:0);
}
// Write hop count filters
@@ -174,7 +175,7 @@ public class JobManager implements IJobM
throws java.io.IOException, ManifoldCFException
{
int version = ManifoldCF.readDword(is);
- if (version != 2)
+ if (version != 2 && version != 3)
throw new java.io.IOException("Unknown job configuration version: "+Integer.toString(version));
int count = ManifoldCF.readDword(is);
int i = 0;
@@ -208,9 +209,14 @@ public class JobManager implements IJobM
EnumeratedValues minutesOfHour = readEnumeratedValues(is);
String timezone = ManifoldCF.readString(is);
Long duration = ManifoldCF.readLong(is);
+ boolean requestMinimum;
+ if (version >= 3)
+ requestMinimum = (ManifoldCF.readByte(is) != 0);
+ else
+ requestMinimum = false;
ScheduleRecord sr = new ScheduleRecord(dayOfWeek, monthOfYear, dayOfMonth, year,
- hourOfDay, minutesOfHour, timezone, duration);
+ hourOfDay, minutesOfHour, timezone, duration, requestMinimum);
job.addScheduleRecord(sr);
j++;
}
@@ -1343,11 +1349,12 @@ public class JobManager implements IJobM
sb.append("SELECT t0.").append(jobQueue.docHashField).append(" FROM ").append(jobQueue.getTableName()).append(" t0 WHERE ")
.append(database.buildConjunctionClause(list,new ClauseDescription[]{
new MultiClause("t0."+jobQueue.docHashField,docList)})).append(" AND ")
- .append("t0.").append(jobQueue.statusField).append(" IN (?,?,?,?) AND ");
+ .append("t0.").append(jobQueue.statusField).append(" IN (?,?,?,?,?) AND ");
list.add(jobQueue.statusToString(jobQueue.STATUS_PURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_PENDINGPURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_COMPLETE));
+ list.add(jobQueue.statusToString(jobQueue.STATUS_UNCHANGED));
list.add(jobQueue.statusToString(jobQueue.STATUS_ELIGIBLEFORDELETE));
sb.append("EXISTS(SELECT 'x' FROM ").append(jobs.getTableName()).append(" t1 WHERE ")
@@ -1424,6 +1431,7 @@ public class JobManager implements IJobM
sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{
new MultiClause(jobQueue.statusField,new Object[]{
jobQueue.statusToString(JobQueue.STATUS_COMPLETE),
+ jobQueue.statusToString(JobQueue.STATUS_UNCHANGED),
jobQueue.statusToString(JobQueue.STATUS_PURGATORY)}),
new UnitaryClause(jobQueue.prioritySetField,"<",new Long(currentTime))})).append(" ");
@@ -1490,6 +1498,7 @@ public class JobManager implements IJobM
.append(database.buildConjunctionClause(list,new ClauseDescription[]{
new MultiClause("t1."+jobs.statusField,new Object[]{
Jobs.statusToString(Jobs.STATUS_STARTINGUP),
+ Jobs.statusToString(Jobs.STATUS_STARTINGUPMINIMAL),
Jobs.statusToString(Jobs.STATUS_ACTIVE),
Jobs.statusToString(Jobs.STATUS_ACTIVESEEDING),
Jobs.statusToString(Jobs.STATUS_ACTIVE_UNINSTALLED),
@@ -4745,13 +4754,14 @@ public class JobManager implements IJobM
// We go through *all* the schedule records. The one that matches that has the latest
// end time is the one we take.
- int l = 0;
Long matchTime = null;
Long duration = null;
- while (l < thisSchedule.length)
+ boolean requestMinimum = false;
+
+ for (int l = 0; l < thisSchedule.length; l++)
{
long trialStartInterval = startInterval;
- ScheduleRecord sr = thisSchedule[l++];
+ ScheduleRecord sr = thisSchedule[l];
Long thisDuration = sr.getDuration();
if (startMethod == IJobDescription.START_WINDOWINSIDE &&
thisDuration != null)
@@ -4787,10 +4797,11 @@ public class JobManager implements IJobM
if (matchTime == null || thisDuration == null ||
(duration != null && thisMatchTime.longValue() + thisDuration.longValue() >
- matchTime.longValue() + duration.longValue()))
+ matchTime.longValue() + duration.longValue()))
{
matchTime = thisMatchTime;
duration = thisDuration;
+ requestMinimum = sr.getRequestMinimum();
}
}
@@ -4823,7 +4834,7 @@ public class JobManager implements IJobM
// If job was formerly "inactive", do the full startup.
// Start this job! but with no end time.
// This does not get logged because the startup thread does the logging.
- jobs.startJob(jobID,windowEnd);
+ jobs.startJob(jobID,windowEnd,requestMinimum);
jobQueue.clearFailTimes(jobID);
if (Logging.jobs.isDebugEnabled())
{
@@ -5256,6 +5267,18 @@ public class JobManager implements IJobM
public void manualStart(Long jobID)
throws ManifoldCFException
{
+ manualStart(jobID,false);
+ }
+
+ /** Manually start a job. The specified job will be run REGARDLESS of the timed windows, and
+ * will not cease until complete. If the job is already running, this operation will assure that
+ * the job does not pause when its window ends. The job can be manually paused, or manually aborted.
+ *@param jobID is the ID of the job to start.
+ *@param requestMinimum is true if a minimal job run is requested.
+ */
+ public void manualStart(Long jobID, boolean requestMinimum)
+ throws ManifoldCFException
+ {
database.beginTransaction();
try
{
@@ -5284,8 +5307,9 @@ public class JobManager implements IJobM
Logging.jobs.debug("Manually starting job "+jobID);
}
// Start this job! but with no end time.
- jobs.startJob(jobID,null);
+ jobs.startJob(jobID,null,requestMinimum);
jobQueue.clearFailTimes(jobID);
+
if (Logging.jobs.isDebugEnabled())
{
Logging.jobs.debug("Manual job start signal for job "+jobID+" successfully sent");
@@ -5355,12 +5379,71 @@ public class JobManager implements IJobM
jobQueue.prepareDeleteScan(jobID);
}
+ /** Prepare a job to be run.
+ * This method is called regardless of the details of the job; what differs is only the flags that are passed in.
+ * The code inside will determine the appropriate procedures.
+ * (This method replaces prepareFullScan() and prepareIncrementalScan(). )
+ *@param jobID is the job id.
+ *@param legalLinkTypes are the link types allowed for the job.
+ *@param hopcountMethod describes how to handle deletions for hopcount purposes.
+ *@param connectorModel is the model used by the connector for the job.
+ *@param continuousJob is true if the job is a continuous one.
+ *@param fromBeginningOfTime is true if the job is running starting from time 0.
+ *@param requestMinimum is true if the minimal amount of work is requested for the job run.
+ */
+ public void prepareJobScan(Long jobID, String[] legalLinkTypes, int hopcountMethod,
+ int connectorModel, boolean continuousJob, boolean fromBeginningOfTime,
+ boolean requestMinimum)
+ throws ManifoldCFException
+ {
+ // (1) If the connector has MODEL_ADD_CHANGE_DELETE, then
+ // we let the connector run the show; there's no purge phase, and therefore the
+ // documents are left in a COMPLETED state if they don't show up in the list
+ // of seeds that require the attention of the connector.
+ //
+ // (2) If the connector has MODEL_ALL, then it's a full crawl no matter what, so
+ // we do a full scan initialization.
+ //
+ // (3) If the connector has some other model, we look at the start time. A start
+ // time of 0 implies a full scan, while any other start time implies an incremental
+ // scan.
+
+ // Complete connector model is told everything, so no delete phase.
+ if (connectorModel == IRepositoryConnector.MODEL_ADD_CHANGE_DELETE)
+ return;
+
+ // If the connector model is complete via chaining, then we just need to make
+ // sure discovery works to queue the changes.
+ if (connectorModel == IRepositoryConnector.MODEL_CHAINED_ADD_CHANGE_DELETE)
+ {
+ jobQueue.preparePartialScan(jobID);
+ return;
+ }
+
+ // Similarly, minimal crawl attempts no delete phase unless the connector explicitly forbids it, or unless
+ // the job criteria have changed.
+ if (requestMinimum && connectorModel != IRepositoryConnector.MODEL_ALL && !fromBeginningOfTime)
+ {
+ // If it is a chained model, do the partial prep.
+ if (connectorModel == IRepositoryConnector.MODEL_CHAINED_ADD ||
+ connectorModel == IRepositoryConnector.MODEL_CHAINED_ADD_CHANGE)
+ jobQueue.preparePartialScan(jobID);
+ return;
+ }
+
+ if (!continuousJob && connectorModel != IRepositoryConnector.MODEL_PARTIAL &&
+ (connectorModel == IRepositoryConnector.MODEL_ALL || fromBeginningOfTime))
+ prepareFullScan(jobID,legalLinkTypes,hopcountMethod);
+ else
+ jobQueue.prepareIncrementalScan(jobID);
+ }
+
/** Prepare for a full scan.
*@param jobID is the job id.
*@param legalLinkTypes are the link types allowed for the job.
*@param hopcountMethod describes how to handle deletions for hopcount purposes.
*/
- public void prepareFullScan(Long jobID, String[] legalLinkTypes, int hopcountMethod)
+ protected void prepareFullScan(Long jobID, String[] legalLinkTypes, int hopcountMethod)
throws ManifoldCFException
{
while (true)
@@ -5370,7 +5453,7 @@ public class JobManager implements IJobM
database.beginTransaction(database.TRANSACTION_SERIALIZED);
try
{
- // Delete all documents that match a given criteria
+ // Delete the documents we have never fetched, including any hopcount records we've calculated.
if (legalLinkTypes.length > 0)
{
ArrayList list = new ArrayList();
@@ -5413,17 +5496,6 @@ public class JobManager implements IJobM
}
}
- /** Prepare for an incremental scan.
- *@param jobID is the job id.
- *@param legalLinkTypes are the link types allowed for the job.
- *@param hopcountMethod describes how to handle deletions for hopcount purposes.
- */
- public void prepareIncrementalScan(Long jobID, String[] legalLinkTypes, int hopcountMethod)
- throws ManifoldCFException
- {
- jobQueue.prepareIncrementalScan(jobID);
- }
-
/** Manually abort a running job. The job will be permanently stopped, and will not run again until
* automatically started based on schedule, or manually started.
*@param jobID is the job to abort.
@@ -5479,8 +5551,9 @@ public class JobManager implements IJobM
/** Manually restart a running job. The job will be stopped and restarted. Any schedule affinity will be lost,
* until the job finishes on its own.
*@param jobID is the job to abort.
+ *@param requestMinimum is true if a minimal job run is requested.
*/
- public void manualAbortRestart(Long jobID)
+ public void manualAbortRestart(Long jobID, boolean requestMinimum)
throws ManifoldCFException
{
if (Logging.jobs.isDebugEnabled())
@@ -5493,7 +5566,7 @@ public class JobManager implements IJobM
database.beginTransaction();
try
{
- jobs.abortRestartJob(jobID);
+ jobs.abortRestartJob(jobID,requestMinimum);
database.performCommit();
break;
}
@@ -5526,6 +5599,16 @@ public class JobManager implements IJobM
}
}
+ /** Manually restart a running job. The job will be stopped and restarted. Any schedule affinity will be lost,
+ * until the job finishes on its own.
+ *@param jobID is the job to abort.
+ */
+ public void manualAbortRestart(Long jobID)
+ throws ManifoldCFException
+ {
+ manualAbortRestart(jobID,false);
+ }
+
/** Abort a running job due to a fatal error condition.
*@param jobID is the job to abort.
*@param errorText is the error text.
@@ -5687,7 +5770,7 @@ public class JobManager implements IJobM
*@return jobs that are active and are running in adaptive mode. These will be seeded
* based on what the connector says should be added to the queue.
*/
- public JobStartRecord[] getJobsReadyForSeeding(long currentTime)
+ public JobSeedingRecord[] getJobsReadyForSeeding(long currentTime)
throws ManifoldCFException
{
while (true)
@@ -5715,7 +5798,7 @@ public class JobManager implements IJobM
IResultSet set = database.performQuery(sb.toString(),list,null,null);
// Update them all
- JobStartRecord[] rval = new JobStartRecord[set.getRowCount()];
+ JobSeedingRecord[] rval = new JobSeedingRecord[set.getRowCount()];
int i = 0;
while (i < rval.length)
{
@@ -5741,7 +5824,7 @@ public class JobManager implements IJobM
Logging.jobs.debug("Marked job "+jobID+" for seeding");
}
- rval[i] = new JobStartRecord(jobID,synchTime);
+ rval[i] = new JobSeedingRecord(jobID,synchTime);
i++;
}
database.performCommit();
@@ -5775,7 +5858,7 @@ public class JobManager implements IJobM
/** Get the list of jobs that are ready for deletion.
*@return jobs that were in the "readyfordelete" state.
*/
- public JobStartRecord[] getJobsReadyForDelete()
+ public JobDeleteRecord[] getJobsReadyForDelete()
throws ManifoldCFException
{
while (true)
@@ -5795,7 +5878,7 @@ public class JobManager implements IJobM
IResultSet set = database.performQuery(sb.toString(),list,null,null);
// Update them all
- JobStartRecord[] rval = new JobStartRecord[set.getRowCount()];
+ JobDeleteRecord[] rval = new JobDeleteRecord[set.getRowCount()];
int i = 0;
while (i < rval.length)
{
@@ -5809,7 +5892,7 @@ public class JobManager implements IJobM
Logging.jobs.debug("Marked job "+jobID+" for delete startup");
}
- rval[i] = new JobStartRecord(jobID,0L);
+ rval[i] = new JobDeleteRecord(jobID);
i++;
}
database.performCommit();
@@ -5857,10 +5940,13 @@ public class JobManager implements IJobM
ArrayList list = new ArrayList();
sb.append(jobs.idField).append(",")
- .append(jobs.lastCheckTimeField)
+ .append(jobs.lastCheckTimeField).append(",")
+ .append(jobs.statusField)
.append(" FROM ").append(jobs.getTableName()).append(" WHERE ")
.append(database.buildConjunctionClause(list,new ClauseDescription[]{
- new UnitaryClause(jobs.statusField,jobs.statusToString(jobs.STATUS_READYFORSTARTUP))}))
+ new MultiClause(jobs.statusField,new Object[]{
+ jobs.statusToString(jobs.STATUS_READYFORSTARTUP),
+ jobs.statusToString(jobs.STATUS_READYFORSTARTUPMINIMAL)})}))
.append(" FOR UPDATE");
IResultSet set = database.performQuery(sb.toString(),list,null,null);
@@ -5872,18 +5958,22 @@ public class JobManager implements IJobM
IResultRow row = set.getRow(i);
Long jobID = (Long)row.getValue(jobs.idField);
Long x = (Long)row.getValue(jobs.lastCheckTimeField);
+ int status = jobs.stringToStatus((String)row.getValue(jobs.statusField));
+
+ boolean requestMinimum = (status == jobs.STATUS_READYFORSTARTUPMINIMAL);
+
long synchTime = 0;
if (x != null)
synchTime = x.longValue();
// Mark status of job as "starting"
- jobs.writeStatus(jobID,jobs.STATUS_STARTINGUP);
+ jobs.writeStatus(jobID,requestMinimum?jobs.STATUS_STARTINGUPMINIMAL:jobs.STATUS_STARTINGUP);
if (Logging.jobs.isDebugEnabled())
{
Logging.jobs.debug("Marked job "+jobID+" for startup");
}
- rval[i] = new JobStartRecord(jobID,synchTime);
+ rval[i] = new JobStartRecord(jobID,synchTime,requestMinimum);
i++;
}
database.performCommit();
@@ -6151,7 +6241,15 @@ public class JobManager implements IJobM
// Set the state of the job back to "ReadyForStartup"
jobs.writeStatus(jobID,jobs.STATUS_READYFORSTARTUP);
break;
+ case Jobs.STATUS_STARTINGUPMINIMAL:
+ if (Logging.jobs.isDebugEnabled())
+ Logging.jobs.debug("Setting job "+jobID+" back to 'ReadyForStartupMinimal' state");
+
+ // Set the state of the job back to "ReadyForStartupMinimal"
+ jobs.writeStatus(jobID,jobs.STATUS_READYFORSTARTUPMINIMAL);
+ break;
case Jobs.STATUS_ABORTINGSTARTINGUP:
+ case Jobs.STATUS_ABORTINGSTARTINGUPMINIMAL:
if (Logging.jobs.isDebugEnabled())
Logging.jobs.debug("Setting job "+jobID+" to 'Aborting' state");
jobs.writeStatus(jobID,jobs.STATUS_ABORTING);
@@ -6161,10 +6259,17 @@ public class JobManager implements IJobM
Logging.jobs.debug("Setting job "+jobID+" to 'AbortingForRestart' state");
jobs.writeStatus(jobID,jobs.STATUS_ABORTINGFORRESTART);
break;
+ case Jobs.STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:
+ if (Logging.jobs.isDebugEnabled())
+ Logging.jobs.debug("Setting job "+jobID+" to 'AbortingForRestartMinimal' state");
+ jobs.writeStatus(jobID,jobs.STATUS_ABORTINGFORRESTARTMINIMAL);
+ break;
case Jobs.STATUS_READYFORSTARTUP:
+ case Jobs.STATUS_READYFORSTARTUPMINIMAL:
case Jobs.STATUS_ABORTING:
case Jobs.STATUS_ABORTINGFORRESTART:
+ case Jobs.STATUS_ABORTINGFORRESTARTMINIMAL:
// ok
break;
default:
@@ -6291,8 +6396,17 @@ public class JobManager implements IJobM
jobs.writeStatus(jobID,jobs.STATUS_ABORTINGFORRESTART);
break;
+ case Jobs.STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:
+ if (Logging.jobs.isDebugEnabled())
+ Logging.jobs.debug("Setting job "+jobID+" back to 'AbortingForRestartMinimal' state");
+
+ // Set the state of the job back to "Active"
+ jobs.writeStatus(jobID,jobs.STATUS_ABORTINGFORRESTARTMINIMAL);
+ break;
+
case Jobs.STATUS_ABORTING:
case Jobs.STATUS_ABORTINGFORRESTART:
+ case Jobs.STATUS_ABORTINGFORRESTARTMINIMAL:
case Jobs.STATUS_ACTIVE:
case Jobs.STATUS_ACTIVE_UNINSTALLED:
case Jobs.STATUS_ACTIVE_NOOUTPUT:
@@ -6544,7 +6658,7 @@ public class JobManager implements IJobM
/** Find the list of jobs that need to have their connectors notified of job completion.
*@return the ID's of jobs that need their output connectors notified in order to become inactive.
*/
- public JobStartRecord[] getJobsReadyForInactivity()
+ public JobNotifyRecord[] getJobsReadyForInactivity()
throws ManifoldCFException
{
while (true)
@@ -6564,7 +6678,7 @@ public class JobManager implements IJobM
IResultSet set = database.performQuery(sb.toString(),list,null,null);
// Return them all
- JobStartRecord[] rval = new JobStartRecord[set.getRowCount()];
+ JobNotifyRecord[] rval = new JobNotifyRecord[set.getRowCount()];
int i = 0;
while (i < rval.length)
{
@@ -6576,7 +6690,7 @@ public class JobManager implements IJobM
{
Logging.jobs.debug("Found job "+jobID+" in need of notification");
}
- rval[i++] = new JobStartRecord(jobID,0L);
+ rval[i++] = new JobNotifyRecord(jobID);
}
database.performCommit();
return rval;
@@ -6672,6 +6786,7 @@ public class JobManager implements IJobM
new MultiClause(jobs.statusField,new Object[]{
jobs.statusToString(jobs.STATUS_ABORTING),
jobs.statusToString(jobs.STATUS_ABORTINGFORRESTART),
+ jobs.statusToString(jobs.STATUS_ABORTINGFORRESTARTMINIMAL),
jobs.statusToString(jobs.STATUS_PAUSING),
jobs.statusToString(jobs.STATUS_PAUSINGSEEDING),
jobs.statusToString(jobs.STATUS_ACTIVEWAITING),
@@ -7019,6 +7134,7 @@ public class JobManager implements IJobM
.append(database.buildConjunctionClause(list,new ClauseDescription[]{
new MultiClause(JobQueue.statusField,new Object[]{
JobQueue.statusToString(JobQueue.STATUS_COMPLETE),
+ JobQueue.statusToString(JobQueue.STATUS_UNCHANGED),
JobQueue.statusToString(JobQueue.STATUS_PURGATORY),
JobQueue.statusToString(JobQueue.STATUS_ACTIVEPURGATORY),
JobQueue.statusToString(JobQueue.STATUS_ACTIVENEEDRESCANPURGATORY),
@@ -7131,11 +7247,15 @@ public class JobManager implements IJobM
case Jobs.STATUS_ABORTING:
case Jobs.STATUS_ABORTINGSEEDING:
case Jobs.STATUS_ABORTINGSTARTINGUP:
+ case Jobs.STATUS_ABORTINGSTARTINGUPMINIMAL:
rstatus = JobStatus.JOBSTATUS_ABORTING;
break;
case Jobs.STATUS_ABORTINGFORRESTART:
+ case Jobs.STATUS_ABORTINGFORRESTARTMINIMAL:
case Jobs.STATUS_ABORTINGFORRESTARTSEEDING:
+ case Jobs.STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:
case Jobs.STATUS_ABORTINGSTARTINGUPFORRESTART:
+ case Jobs.STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:
rstatus = JobStatus.JOBSTATUS_RESTARTING;
break;
case Jobs.STATUS_PAUSING:
@@ -7163,7 +7283,9 @@ public class JobManager implements IJobM
rstatus = JobStatus.JOBSTATUS_PAUSED;
break;
case Jobs.STATUS_STARTINGUP:
+ case Jobs.STATUS_STARTINGUPMINIMAL:
case Jobs.STATUS_READYFORSTARTUP:
+ case Jobs.STATUS_READYFORSTARTUPMINIMAL:
rstatus = JobStatus.JOBSTATUS_STARTING;
break;
case Jobs.STATUS_DELETESTARTINGUP:
@@ -7221,6 +7343,7 @@ public class JobManager implements IJobM
.append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Processed'")
.append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Processed'")
.append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Processed'")
+ .append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Processed'")
.append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Being removed'")
.append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Being removed'")
.append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Being removed'")
@@ -7229,7 +7352,7 @@ public class JobManager implements IJobM
.append(" END AS state,")
.append("CASE")
.append(" WHEN ")
- .append("t0.").append(jobQueue.statusField).append(" IN (?,?)")
+ .append("t0.").append(jobQueue.statusField).append(" IN (?,?,?)")
.append(" THEN 'Inactive'")
.append(" WHEN ")
.append("t0.").append(jobQueue.statusField).append(" IN (?,?)")
@@ -7289,6 +7412,7 @@ public class JobManager implements IJobM
list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_COMPLETE));
+ list.add(jobQueue.statusToString(jobQueue.STATUS_UNCHANGED));
list.add(jobQueue.statusToString(jobQueue.STATUS_PURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED));
list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED));
@@ -7296,6 +7420,7 @@ public class JobManager implements IJobM
list.add(jobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVED));
list.add(jobQueue.statusToString(jobQueue.STATUS_COMPLETE));
+ list.add(jobQueue.statusToString(jobQueue.STATUS_UNCHANGED));
list.add(jobQueue.statusToString(jobQueue.STATUS_PURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_PENDING));
@@ -7378,7 +7503,7 @@ public class JobManager implements IJobM
sb.append(" AS idbucket,")
.append("CASE")
.append(" WHEN ")
- .append(jobQueue.statusField).append(" IN (?,?)")
+ .append(jobQueue.statusField).append(" IN (?,?,?)")
.append(" THEN 1 ELSE 0")
.append(" END")
.append(" AS inactive,")
@@ -7450,6 +7575,7 @@ public class JobManager implements IJobM
sb.append(" FROM ").append(jobQueue.getTableName());
list.add(jobQueue.statusToString(jobQueue.STATUS_COMPLETE));
+ list.add(jobQueue.statusToString(jobQueue.STATUS_UNCHANGED));
list.add(jobQueue.statusToString(jobQueue.STATUS_PURGATORY));
list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVE));
@@ -7576,6 +7702,7 @@ public class JobManager implements IJobM
jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED),
jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED),
jobQueue.statusToString(jobQueue.STATUS_COMPLETE),
+ jobQueue.statusToString(jobQueue.STATUS_UNCHANGED),
jobQueue.statusToString(jobQueue.STATUS_PURGATORY)})}));
break;
case DOCSTATE_OUTOFSCOPE:
@@ -7602,6 +7729,7 @@ public class JobManager implements IJobM
sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{
new MultiClause(fieldPrefix+jobQueue.statusField,new Object[]{
jobQueue.statusToString(jobQueue.STATUS_COMPLETE),
+ jobQueue.statusToString(jobQueue.STATUS_UNCHANGED),
jobQueue.statusToString(jobQueue.STATUS_PURGATORY)})}));
break;
case DOCSTATUS_PROCESSING:
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java Sat Mar 16 21:03:08 2013
@@ -60,15 +60,17 @@ public class JobQueue extends org.apache
public final static int STATUS_PENDING = 0;
public final static int STATUS_ACTIVE = 1;
public final static int STATUS_COMPLETE = 2;
- public final static int STATUS_PENDINGPURGATORY = 3;
- public final static int STATUS_ACTIVEPURGATORY = 4;
- public final static int STATUS_PURGATORY = 5;
- public final static int STATUS_BEINGDELETED = 6;
- public final static int STATUS_ACTIVENEEDRESCAN = 7;
- public final static int STATUS_ACTIVENEEDRESCANPURGATORY = 8;
- public final static int STATUS_BEINGCLEANED = 9;
- public final static int STATUS_ELIGIBLEFORDELETE = 10;
- public final static int STATUS_HOPCOUNTREMOVED = 11;
+ public final static int STATUS_UNCHANGED = 3;
+ public final static int STATUS_PENDINGPURGATORY = 4;
+ public final static int STATUS_ACTIVEPURGATORY = 5;
+ public final static int STATUS_PURGATORY = 6;
+ public final static int STATUS_BEINGDELETED = 7;
+ public final static int STATUS_ACTIVENEEDRESCAN = 8;
+ public final static int STATUS_ACTIVENEEDRESCANPURGATORY = 9;
+ public final static int STATUS_BEINGCLEANED = 10;
+ public final static int STATUS_ELIGIBLEFORDELETE = 11;
+ public final static int STATUS_HOPCOUNTREMOVED = 12;
+
// Action values
public final static int ACTION_RESCAN = 0;
public final static int ACTION_REMOVE = 1;
@@ -121,6 +123,7 @@ public class JobQueue extends org.apache
statusMap.put("P",new Integer(STATUS_PENDING));
statusMap.put("A",new Integer(STATUS_ACTIVE));
statusMap.put("C",new Integer(STATUS_COMPLETE));
+ statusMap.put("U",new Integer(STATUS_UNCHANGED));
statusMap.put("G",new Integer(STATUS_PENDINGPURGATORY));
statusMap.put("F",new Integer(STATUS_ACTIVEPURGATORY));
statusMap.put("Z",new Integer(STATUS_PURGATORY));
@@ -533,6 +536,7 @@ public class JobQueue extends org.apache
new MultiClause(statusField,new Object[]{
statusToString(STATUS_PENDINGPURGATORY),
statusToString(STATUS_COMPLETE),
+ statusToString(STATUS_UNCHANGED),
statusToString(STATUS_PURGATORY)})});
performUpdate(map,"WHERE "+query,list,null);
@@ -583,6 +587,7 @@ public class JobQueue extends org.apache
new UnitaryClause(jobIDField,jobID),
new MultiClause(statusField,new Object[]{
statusToString(STATUS_PENDINGPURGATORY),
+ statusToString(STATUS_UNCHANGED),
statusToString(STATUS_COMPLETE)})});
performUpdate(map,"WHERE "+query,list,null);
@@ -592,6 +597,34 @@ public class JobQueue extends org.apache
unconditionallyAnalyzeTables();
}
+ /** Prepare for a "partial" job. This is called ONLY when the job is inactive.
+ *
+ * This method maps all COMPLETE entries to UNCHANGED. The purpose is to
+ * allow discovery to find the documents that need to be processed. If they were
+ * marked as COMPLETE that would stop them from being queued.
+ *@param jobID is the job identifier.
+ */
+ public void preparePartialScan(Long jobID)
+ throws ManifoldCFException
+ {
+ // Map COMPLETE to UNCHANGED.
+ HashMap map = new HashMap();
+ map.put(statusField,statusToString(STATUS_UNCHANGED));
+ // Do not reset priorities here! They should all be blank at this point.
+ map.put(checkTimeField,new Long(0L));
+ map.put(checkActionField,actionToString(ACTION_RESCAN));
+ map.put(failTimeField,null);
+ map.put(failCountField,null);
+ ArrayList list = new ArrayList();
+ String query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(jobIDField,jobID),
+ new UnitaryClause(statusField,statusToString(STATUS_COMPLETE))});
+ performUpdate(map,"WHERE "+query,list,null);
+ noteModifications(0,1,0);
+ // Do an analyze, otherwise our plans are going to be crap right off the bat
+ unconditionallyAnalyzeTables();
+ }
+
/** Prepare for an "incremental" job. This is called ONLY when the job is inactive;
* that is, there should be no ACTIVE or ACTIVEPURGATORY entries at all.
*
@@ -603,7 +636,7 @@ public class JobQueue extends org.apache
public void prepareIncrementalScan(Long jobID)
throws ManifoldCFException
{
- // Delete PENDING and ACTIVE entries
+ // Map COMPLETE to PENDINGPURGATORY.
HashMap map = new HashMap();
map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
// Do not reset priorities here! They should all be blank at this point.
@@ -614,7 +647,9 @@ public class JobQueue extends org.apache
ArrayList list = new ArrayList();
String query = buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(jobIDField,jobID),
- new UnitaryClause(statusField,statusToString(STATUS_COMPLETE))});
+ new MultiClause(statusField,new Object[]{
+ statusToString(STATUS_COMPLETE),
+ statusToString(STATUS_UNCHANGED)})});
performUpdate(map,"WHERE "+query,list,null);
noteModifications(0,1,0);
// Do an analyze, otherwise our plans are going to be crap right off the bat
@@ -1001,6 +1036,7 @@ public class JobQueue extends org.apache
break;
case STATUS_COMPLETE:
+ case STATUS_UNCHANGED:
case STATUS_PURGATORY:
// Set the status and time both
map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
@@ -1278,6 +1314,7 @@ public class JobQueue extends org.apache
switch (currentStatus)
{
case STATUS_PURGATORY:
+ case STATUS_UNCHANGED:
// Set the status and time both
map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
map.put(checkTimeField,new Long(desiredExecuteTime));
@@ -1506,6 +1543,8 @@ public class JobQueue extends org.apache
return "A";
case STATUS_COMPLETE:
return "C";
+ case STATUS_UNCHANGED:
+ return "U";
case STATUS_PENDINGPURGATORY:
return "G";
case STATUS_ACTIVEPURGATORY:
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java Sat Mar 16 21:03:08 2013
@@ -92,17 +92,23 @@ public class Jobs extends org.apache.man
public static final int STATUS_RESUMINGSEEDING = 17; // In the process of resuming a paused or waited job, seeding process active too; will enter STATUS_ACTIVESEEDING when done.
public static final int STATUS_ABORTING = 18; // Aborting (not yet aborted because documents still being processed)
public static final int STATUS_STARTINGUP = 19; // Loading the queue (will go into ACTIVE if successful, or INACTIVE if not)
- public static final int STATUS_ABORTINGSTARTINGUP = 20; // Will abort once the queue loading is complete
- public static final int STATUS_READYFORSTARTUP = 21; // Job is marked for startup; startup thread has not taken it yet.
- public static final int STATUS_READYFORDELETE = 22; // Job is marked for delete; delete thread has not taken it yet.
- public static final int STATUS_ABORTINGSEEDING = 23; // Same as aborting, but seeding process is currently active also.
- public static final int STATUS_ABORTINGFORRESTART = 24; // Same as aborting, except after abort is complete startup will happen.
- public static final int STATUS_ABORTINGFORRESTARTSEEDING = 25; // Seeding version of aborting for restart
- public static final int STATUS_ABORTINGSTARTINGUPFORRESTART = 26; // Starting up version of aborting for restart
- public static final int STATUS_READYFORNOTIFY = 27; // Job is ready to be notified of completion
- public static final int STATUS_NOTIFYINGOFCOMPLETION = 28; // Notifying connector of terminating job (either aborted, or finished)
- public static final int STATUS_DELETING = 29; // The job is deleting.
- public static final int STATUS_DELETESTARTINGUP = 30; // The delete is starting up.
+ public static final int STATUS_STARTINGUPMINIMAL = 20; // Loading the queue for minimal job run (will go into ACTIVE if successful, or INACTIVE if not)
+ public static final int STATUS_ABORTINGSTARTINGUP = 21; // Will abort once the queue loading is complete
+ public static final int STATUS_ABORTINGSTARTINGUPMINIMAL = 22; // Will abort once the queue loading is complete
+ public static final int STATUS_READYFORSTARTUP = 23; // Job is marked for minimal startup; startup thread has not taken it yet.
+ public static final int STATUS_READYFORSTARTUPMINIMAL = 24; // Job is marked for startup; startup thread has not taken it yet.
+ public static final int STATUS_READYFORDELETE = 25; // Job is marked for delete; delete thread has not taken it yet.
+ public static final int STATUS_ABORTINGSEEDING = 26; // Same as aborting, but seeding process is currently active also.
+ public static final int STATUS_ABORTINGFORRESTART = 27; // Same as aborting, except after abort is complete startup will happen.
+ public static final int STATUS_ABORTINGFORRESTARTMINIMAL = 28; // Same as aborting, except after abort is complete startup will happen.
+ public static final int STATUS_ABORTINGFORRESTARTSEEDING = 29; // Seeding version of aborting for restart
+ public static final int STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL = 30; // Seeding version of aborting for restart
+ public static final int STATUS_ABORTINGSTARTINGUPFORRESTART = 31; // Starting up version of aborting for restart
+ public static final int STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL = 32; // Starting up version of aborting for restart
+ public static final int STATUS_READYFORNOTIFY = 33; // Job is ready to be notified of completion
+ public static final int STATUS_NOTIFYINGOFCOMPLETION = 34; // Notifying connector of terminating job (either aborted, or finished)
+ public static final int STATUS_DELETING = 35; // The job is deleting.
+ public static final int STATUS_DELETESTARTINGUP = 36; // The delete is starting up.
// These statuses have to do with whether a job has an installed underlying connector or not.
// There are two reasons to have a special state here: (1) if the behavior of the crawler differs, or (2) if the
@@ -113,13 +119,13 @@ public class Jobs extends org.apache.man
// But, since there is no indication in the jobs table of an uninstalled connector for such jobs, the code which starts
// jobs up (or otherwise would enter any state that has a corresponding special state) must check to see if the underlying
// connector exists before deciding what state to put the job into.
- public static final int STATUS_ACTIVE_UNINSTALLED = 31; // Active, but repository connector not installed
- public static final int STATUS_ACTIVESEEDING_UNINSTALLED = 32; // Active and seeding, but repository connector not installed
- public static final int STATUS_ACTIVE_NOOUTPUT = 33; // Active, but output connector not installed
- public static final int STATUS_ACTIVESEEDING_NOOUTPUT = 34; // Active and seeding, but output connector not installed
- public static final int STATUS_ACTIVE_NEITHER = 35; // Active, but neither repository connector nor output connector installed
- public static final int STATUS_ACTIVESEEDING_NEITHER = 36; // Active and seeding, but neither repository connector nor output connector installed
- public static final int STATUS_DELETING_NOOUTPUT = 37; // Job is being deleted but there's no output connector installed
+ public static final int STATUS_ACTIVE_UNINSTALLED = 37; // Active, but repository connector not installed
+ public static final int STATUS_ACTIVESEEDING_UNINSTALLED = 38; // Active and seeding, but repository connector not installed
+ public static final int STATUS_ACTIVE_NOOUTPUT = 39; // Active, but output connector not installed
+ public static final int STATUS_ACTIVESEEDING_NOOUTPUT = 40; // Active and seeding, but output connector not installed
+ public static final int STATUS_ACTIVE_NEITHER = 41; // Active, but neither repository connector nor output connector installed
+ public static final int STATUS_ACTIVESEEDING_NEITHER = 42; // Active and seeding, but neither repository connector nor output connector installed
+ public static final int STATUS_DELETING_NOOUTPUT = 43; // Job is being deleted but there's no output connector installed
// Type field values
public static final int TYPE_CONTINUOUS = IJobDescription.TYPE_CONTINUOUS;
@@ -194,13 +200,18 @@ public class Jobs extends org.apache.man
statusMap.put("Z",new Integer(STATUS_PAUSEDWAIT));
statusMap.put("X",new Integer(STATUS_ABORTING));
statusMap.put("B",new Integer(STATUS_STARTINGUP));
+ statusMap.put("b",new Integer(STATUS_STARTINGUPMINIMAL));
statusMap.put("Q",new Integer(STATUS_ABORTINGSTARTINGUP));
+ statusMap.put("q",new Integer(STATUS_ABORTINGSTARTINGUPMINIMAL));
statusMap.put("C",new Integer(STATUS_READYFORSTARTUP));
+ statusMap.put("c",new Integer(STATUS_READYFORSTARTUPMINIMAL));
statusMap.put("E",new Integer(STATUS_READYFORDELETE));
statusMap.put("V",new Integer(STATUS_DELETESTARTINGUP));
statusMap.put("e",new Integer(STATUS_DELETING));
statusMap.put("Y",new Integer(STATUS_ABORTINGFORRESTART));
+ statusMap.put("M",new Integer(STATUS_ABORTINGFORRESTARTMINIMAL));
statusMap.put("T",new Integer(STATUS_ABORTINGSTARTINGUPFORRESTART));
+ statusMap.put("t",new Integer(STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL));
statusMap.put("a",new Integer(STATUS_ACTIVESEEDING));
statusMap.put("x",new Integer(STATUS_ABORTINGSEEDING));
@@ -208,6 +219,7 @@ public class Jobs extends org.apache.man
statusMap.put("w",new Integer(STATUS_ACTIVEWAITSEEDING));
statusMap.put("z",new Integer(STATUS_PAUSEDWAITSEEDING));
statusMap.put("y",new Integer(STATUS_ABORTINGFORRESTARTSEEDING));
+ statusMap.put("m",new Integer(STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL));
statusMap.put("H",new Integer(STATUS_ACTIVEWAITING));
statusMap.put("h",new Integer(STATUS_ACTIVEWAITINGSEEDING));
@@ -842,6 +854,15 @@ public class Jobs extends org.apache.man
map.put(statusField,statusToString(STATUS_READYFORSTARTUP));
performUpdate(map,"WHERE "+query,list,invKey);
+ // Starting up or aborting starting up goes back to just being ready
+ list.clear();
+ query = buildConjunctionClause(list,new ClauseDescription[]{
+ new MultiClause(statusField,new Object[]{
+ statusToString(STATUS_STARTINGUPMINIMAL),
+ statusToString(STATUS_ABORTINGSTARTINGUPMINIMAL)})});
+ map.put(statusField,statusToString(STATUS_READYFORSTARTUPMINIMAL));
+ performUpdate(map,"WHERE "+query,list,invKey);
+
// Aborting starting up for restart state goes to ABORTINGFORRESTART
list.clear();
query = buildConjunctionClause(list,new ClauseDescription[]{
@@ -849,6 +870,13 @@ public class Jobs extends org.apache.man
map.put(statusField,statusToString(STATUS_ABORTINGFORRESTART));
performUpdate(map,"WHERE "+query,list,invKey);
+ // Aborting starting up for restart state goes to ABORTINGFORRESTART
+ list.clear();
+ query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(statusField,statusToString(STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL))});
+ map.put(statusField,statusToString(STATUS_ABORTINGFORRESTARTMINIMAL));
+ performUpdate(map,"WHERE "+query,list,invKey);
+
// All seeding values return to pre-seeding values
list.clear();
query = buildConjunctionClause(list,new ClauseDescription[]{
@@ -887,6 +915,11 @@ public class Jobs extends org.apache.man
performUpdate(map,"WHERE "+query,list,invKey);
list.clear();
query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(statusField,statusToString(STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL))});
+ map.put(statusField,statusToString(STATUS_ABORTINGFORRESTARTMINIMAL));
+ performUpdate(map,"WHERE "+query,list,invKey);
+ list.clear();
+ query = buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(statusField,statusToString(STATUS_PAUSEDSEEDING))});
map.put(statusField,statusToString(STATUS_PAUSED));
performUpdate(map,"WHERE "+query,list,invKey);
@@ -1163,7 +1196,7 @@ public class Jobs extends org.apache.man
int status = stringToStatus(statusValue);
// Any active state in the lifecycle will do: seeding, active, active_seeding
return (status == STATUS_ACTIVE || status == STATUS_ACTIVESEEDING ||
- status == STATUS_STARTINGUP);
+ status == STATUS_STARTINGUP || status == STATUS_STARTINGUPMINIMAL);
}
/** Reset delete startup worker thread status.
@@ -1209,6 +1242,7 @@ public class Jobs extends org.apache.man
HashMap map = new HashMap();
String query;
+ list.clear();
query = buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(statusField,statusToString(STATUS_STARTINGUP))});
map.put(statusField,statusToString(STATUS_READYFORSTARTUP));
@@ -1216,7 +1250,15 @@ public class Jobs extends org.apache.man
list.clear();
query = buildConjunctionClause(list,new ClauseDescription[]{
- new UnitaryClause(statusField,statusToString(STATUS_ABORTINGSTARTINGUP))});
+ new UnitaryClause(statusField,statusToString(STATUS_STARTINGUPMINIMAL))});
+ map.put(statusField,statusToString(STATUS_READYFORSTARTUPMINIMAL));
+ performUpdate(map,"WHERE "+query,list,new StringSet(getJobStatusKey()));
+
+ list.clear();
+ query = buildConjunctionClause(list,new ClauseDescription[]{
+ new MultiClause(statusField,new Object[]{
+ statusToString(STATUS_ABORTINGSTARTINGUP),
+ statusToString(STATUS_ABORTINGSTARTINGUPMINIMAL)})});
map.put(statusField,statusToString(STATUS_ABORTING));
performUpdate(map,"WHERE "+query,list,new StringSet(getJobStatusKey()));
@@ -1226,6 +1268,12 @@ public class Jobs extends org.apache.man
map.put(statusField,statusToString(STATUS_ABORTINGFORRESTART));
performUpdate(map,"WHERE "+query,list,new StringSet(getJobStatusKey()));
+ list.clear();
+ query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(statusField,statusToString(STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL))});
+ map.put(statusField,statusToString(STATUS_ABORTINGFORRESTARTMINIMAL));
+ performUpdate(map,"WHERE "+query,list,new StringSet(getJobStatusKey()));
+
}
/** Reset as part of restoring seeding worker threads.
@@ -1278,6 +1326,11 @@ public class Jobs extends org.apache.man
performUpdate(map,"WHERE "+query,list,invKey);
list.clear();
query = buildConjunctionClause(list,new ClauseDescription[]{
+ new UnitaryClause(statusField,statusToString(STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL))});
+ map.put(statusField,statusToString(STATUS_ABORTINGFORRESTARTMINIMAL));
+ performUpdate(map,"WHERE "+query,list,invKey);
+ list.clear();
+ query = buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(statusField,statusToString(STATUS_PAUSEDSEEDING))});
map.put(statusField,statusToString(STATUS_PAUSED));
performUpdate(map,"WHERE "+query,list,invKey);
@@ -1329,15 +1382,16 @@ public class Jobs extends org.apache.man
* when the job enters the "active" state.)
*@param jobID is the job identifier.
*@param windowEnd is the window end time, if any
+ *@param requestMinimum is true if a minimal job run is requested
*/
- public void startJob(Long jobID, Long windowEnd)
+ public void startJob(Long jobID, Long windowEnd, boolean requestMinimum)
throws ManifoldCFException
{
ArrayList list = new ArrayList();
String query = buildConjunctionClause(list,new ClauseDescription[]{
new UnitaryClause(idField,jobID)});
HashMap map = new HashMap();
- map.put(statusField,statusToString(STATUS_READYFORSTARTUP));
+ map.put(statusField,statusToString(requestMinimum?STATUS_READYFORSTARTUPMINIMAL:STATUS_READYFORSTARTUP));
map.put(endTimeField,null);
// Make sure error is removed (from last time)
map.put(errorField,null);
@@ -1488,6 +1542,7 @@ public class Jobs extends org.apache.man
switch (status)
{
case STATUS_STARTINGUP:
+ case STATUS_STARTINGUPMINIMAL:
if (connectionMgr.checkConnectorExists((String)row.getValue(connectionNameField)))
{
if (outputMgr.checkConnectorExists((String)row.getValue(outputNameField)))
@@ -1504,11 +1559,15 @@ public class Jobs extends org.apache.man
}
break;
case STATUS_ABORTINGSTARTINGUP:
+ case STATUS_ABORTINGSTARTINGUPMINIMAL:
newStatus = STATUS_ABORTING;
break;
case STATUS_ABORTINGSTARTINGUPFORRESTART:
newStatus = STATUS_ABORTINGFORRESTART;
break;
+ case STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:
+ newStatus = STATUS_ABORTINGFORRESTARTMINIMAL;
+ break;
default:
// Complain!
throw new ManifoldCFException("Unexpected job status encountered: "+Integer.toString(status));
@@ -1600,6 +1659,9 @@ public class Jobs extends org.apache.man
case STATUS_ABORTINGFORRESTARTSEEDING:
newStatus = STATUS_ABORTINGFORRESTART;
break;
+ case STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:
+ newStatus = STATUS_ABORTINGFORRESTARTMINIMAL;
+ break;
default:
throw new ManifoldCFException("Unexpected job status encountered: "+Integer.toString(status));
}
@@ -1677,7 +1739,8 @@ public class Jobs extends org.apache.man
throw new ManifoldCFException("Job does not exist: "+jobID);
IResultRow row = set.getRow(0);
int status = stringToStatus(row.getValue(statusField).toString());
- if (status == STATUS_ABORTING || status == STATUS_ABORTINGSEEDING || status == STATUS_ABORTINGSTARTINGUP)
+ if (status == STATUS_ABORTING || status == STATUS_ABORTINGSEEDING ||
+ status == STATUS_ABORTINGSTARTINGUP || status == STATUS_ABORTINGSTARTINGUPMINIMAL)
return false;
int newStatus;
switch (status)
@@ -1686,7 +1749,12 @@ public class Jobs extends org.apache.man
case STATUS_ABORTINGSTARTINGUPFORRESTART:
newStatus = STATUS_ABORTINGSTARTINGUP;
break;
+ case STATUS_STARTINGUPMINIMAL:
+ case STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:
+ newStatus = STATUS_ABORTINGSTARTINGUPMINIMAL;
+ break;
case STATUS_READYFORSTARTUP:
+ case STATUS_READYFORSTARTUPMINIMAL:
case STATUS_ACTIVE:
case STATUS_ACTIVE_UNINSTALLED:
case STATUS_ACTIVE_NOOUTPUT:
@@ -1698,6 +1766,7 @@ public class Jobs extends org.apache.man
case STATUS_PAUSED:
case STATUS_PAUSEDWAIT:
case STATUS_ABORTINGFORRESTART:
+ case STATUS_ABORTINGFORRESTARTMINIMAL:
newStatus = STATUS_ABORTING;
break;
case STATUS_ACTIVESEEDING:
@@ -1711,6 +1780,7 @@ public class Jobs extends org.apache.man
case STATUS_PAUSEDSEEDING:
case STATUS_PAUSEDWAITSEEDING:
case STATUS_ABORTINGFORRESTARTSEEDING:
+ case STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:
newStatus = STATUS_ABORTINGSEEDING;
break;
default:
@@ -1726,8 +1796,9 @@ public class Jobs extends org.apache.man
/** Restart a job. Finish off what's currently happening, and then start the job up again.
*@param jobID is the job id.
+ *@param requestMinimum is true if the minimal job run is requested.
*/
- public void abortRestartJob(Long jobID)
+ public void abortRestartJob(Long jobID, boolean requestMinimum)
throws ManifoldCFException
{
// Get the current job status
@@ -1740,15 +1811,20 @@ public class Jobs extends org.apache.man
throw new ManifoldCFException("Job does not exist: "+jobID);
IResultRow row = set.getRow(0);
int status = stringToStatus(row.getValue(statusField).toString());
- if (status == STATUS_ABORTINGFORRESTART || status == STATUS_ABORTINGFORRESTARTSEEDING || status == STATUS_ABORTINGSTARTINGUPFORRESTART)
+ if (status == STATUS_ABORTINGFORRESTART || status == STATUS_ABORTINGFORRESTARTSEEDING ||
+ status == STATUS_ABORTINGSTARTINGUPFORRESTART ||
+ status == STATUS_ABORTINGFORRESTARTMINIMAL || status == STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL ||
+ status == STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL)
return;
int newStatus;
switch (status)
{
case STATUS_STARTINGUP:
- newStatus = STATUS_ABORTINGSTARTINGUPFORRESTART;
+ case STATUS_STARTINGUPMINIMAL:
+ newStatus = requestMinimum?STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:STATUS_ABORTINGSTARTINGUPFORRESTART;
break;
case STATUS_READYFORSTARTUP:
+ case STATUS_READYFORSTARTUPMINIMAL:
case STATUS_ACTIVE:
case STATUS_ACTIVE_UNINSTALLED:
case STATUS_ACTIVE_NOOUTPUT:
@@ -1759,7 +1835,7 @@ public class Jobs extends org.apache.man
case STATUS_PAUSINGWAITING:
case STATUS_PAUSED:
case STATUS_PAUSEDWAIT:
- newStatus = STATUS_ABORTINGFORRESTART;
+ newStatus = requestMinimum?STATUS_ABORTINGFORRESTARTMINIMAL:STATUS_ABORTINGFORRESTART;
break;
case STATUS_ACTIVESEEDING:
case STATUS_ACTIVESEEDING_UNINSTALLED:
@@ -1771,7 +1847,7 @@ public class Jobs extends org.apache.man
case STATUS_PAUSINGWAITINGSEEDING:
case STATUS_PAUSEDSEEDING:
case STATUS_PAUSEDWAITSEEDING:
- newStatus = STATUS_ABORTINGFORRESTARTSEEDING;
+ newStatus = requestMinimum?STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:STATUS_ABORTINGFORRESTARTSEEDING;
break;
default:
throw new ManifoldCFException("Job "+jobID+" is not restartable");
@@ -2061,6 +2137,13 @@ public class Jobs extends org.apache.man
map.put(errorField,null);
map.put(windowEndField,null);
break;
+ case STATUS_ABORTINGFORRESTARTMINIMAL:
+ map.put(statusField,statusToString(STATUS_READYFORSTARTUPMINIMAL));
+ map.put(endTimeField,null);
+ // Make sure error is removed (from last time)
+ map.put(errorField,null);
+ map.put(windowEndField,null);
+ break;
case STATUS_PAUSING:
map.put(statusField,statusToString(STATUS_PAUSED));
break;
@@ -2295,14 +2378,24 @@ public class Jobs extends org.apache.man
return "X";
case STATUS_ABORTINGFORRESTART:
return "Y";
+ case STATUS_ABORTINGFORRESTARTMINIMAL:
+ return "M";
case STATUS_STARTINGUP:
return "B";
+ case STATUS_STARTINGUPMINIMAL:
+ return "b";
case STATUS_ABORTINGSTARTINGUP:
return "Q";
+ case STATUS_ABORTINGSTARTINGUPMINIMAL:
+ return "q";
case STATUS_ABORTINGSTARTINGUPFORRESTART:
return "T";
+ case STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:
+ return "t";
case STATUS_READYFORSTARTUP:
return "C";
+ case STATUS_READYFORSTARTUPMINIMAL:
+ return "c";
case STATUS_READYFORDELETE:
return "E";
case STATUS_DELETESTARTINGUP:
@@ -2321,6 +2414,8 @@ public class Jobs extends org.apache.man
return "z";
case STATUS_ABORTINGFORRESTARTSEEDING:
return "y";
+ case STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:
+ return "m";
case STATUS_ACTIVE_UNINSTALLED:
return "R";
case STATUS_ACTIVESEEDING_UNINSTALLED:
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ScheduleManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ScheduleManager.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ScheduleManager.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ScheduleManager.java Sat Mar 16 21:03:08 2013
@@ -41,6 +41,7 @@ import java.util.*;
* <tr><td>minutesofhour</td><td>VARCHAR(255)</td><td></td></tr>
* <tr><td>timezone</td><td>VARCHAR(32)</td><td></td></tr>
* <tr><td>windowlength</td><td>BIGINT</td><td></td></tr>
+* <tr><td>requestminimum</td><td>CHAR(1)</td><td></td></tr>
* </table>
* <br><br>
*
@@ -60,7 +61,7 @@ public class ScheduleManager extends org
public final static String minutesOfHourField = "minutesofhour";
public final static String timezoneField = "timezone";
public final static String windowDurationField = "windowlength";
-
+ public final static String requestMinimumField = "requestminimum";
/** Constructor.
*@param threadContext is the thread context.
@@ -96,6 +97,7 @@ public class ScheduleManager extends org
map.put(minutesOfHourField,new ColumnDescription("VARCHAR(255)",false,true,null,null,false));
map.put(timezoneField,new ColumnDescription("VARCHAR(32)",false,true,null,null,false));
map.put(windowDurationField,new ColumnDescription("BIGINT",false,true,null,null,false));
+ map.put(requestMinimumField,new ColumnDescription("CHAR(1)",false,true,null,null,false));
performCreate(map,null);
}
else
@@ -113,6 +115,14 @@ public class ScheduleManager extends org
performAlter(null,null,list,null);
}
+
+ if (existing.get(requestMinimumField) == null)
+ {
+ HashMap map = new HashMap();
+ map.put(requestMinimumField,new ColumnDescription("CHAR(1)",false,true,null,null,false));
+ performAlter(map,null,null,null);
+ }
+
}
// Index management
@@ -171,7 +181,8 @@ public class ScheduleManager extends org
stringToEnumeratedValue((String)row.getValue(hourOfDayField)),
stringToEnumeratedValue((String)row.getValue(minutesOfHourField)),
(String)row.getValue(timezoneField),
- (Long)row.getValue(windowDurationField));
+ (Long)row.getValue(windowDurationField),
+ stringToRequestMinimumValue((String)row.getValue(requestMinimumField)));
((JobDescription)returnValues.get(ownerID)).addScheduleRecord(sr);
i++;
}
@@ -209,7 +220,8 @@ public class ScheduleManager extends org
stringToEnumeratedValue((String)row.getValue(hourOfDayField)),
stringToEnumeratedValue((String)row.getValue(minutesOfHourField)),
(String)row.getValue(timezoneField),
- (Long)row.getValue(windowDurationField));
+ (Long)row.getValue(windowDurationField),
+ stringToRequestMinimumValue((String)row.getValue(requestMinimumField)));
ArrayList theList = (ArrayList)returnValues.get(ownerID);
if (theList == null)
{
@@ -245,6 +257,7 @@ public class ScheduleManager extends org
map.put(minutesOfHourField,enumeratedValueToString(record.getMinutesOfHour()));
map.put(timezoneField,record.getTimezone());
map.put(windowDurationField,record.getDuration());
+ map.put(requestMinimumField,requestMinimumValueToString(record.getRequestMinimum()));
map.put(ownerIDField,ownerID);
map.put(ordinalField,new Long((long)i));
performInsert(map,null);
@@ -340,5 +353,22 @@ public class ScheduleManager extends org
return rval.toString();
}
-
+ public static String requestMinimumValueToString(boolean requestMinimum)
+ {
+ return requestMinimum?"T":"F";
+ }
+
+ public static boolean stringToRequestMinimumValue(String requestMinimum)
+ throws ManifoldCFException
+ {
+ if (requestMinimum == null)
+ return false;
+ else if (requestMinimum.equals("T"))
+ return true;
+ else if (requestMinimum.equals("F"))
+ return false;
+ else
+ throw new ManifoldCFException("Bad requestminimum value: '"+requestMinimum+"'");
+ }
+
}
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java Sat Mar 16 21:03:08 2013
@@ -66,7 +66,7 @@ public class JobNotificationThread exten
// Before we begin, conditionally reset
resetManager.waitForReset(threadContext);
- JobStartRecord[] jobsNeedingNotification = jobManager.getJobsReadyForInactivity();
+ JobNotifyRecord[] jobsNeedingNotification = jobManager.getJobsReadyForInactivity();
try
{
HashMap connectionNames = new HashMap();
@@ -74,7 +74,7 @@ public class JobNotificationThread exten
int k = 0;
while (k < jobsNeedingNotification.length)
{
- JobStartRecord jsr = jobsNeedingNotification[k++];
+ JobNotifyRecord jsr = jobsNeedingNotification[k++];
Long jobID = jsr.getJobID();
IJobDescription job = jobManager.load(jobID,true);
if (job != null)
@@ -144,7 +144,7 @@ public class JobNotificationThread exten
k = 0;
while (k < jobsNeedingNotification.length)
{
- JobStartRecord jsr = jobsNeedingNotification[k++];
+ JobNotifyRecord jsr = jobsNeedingNotification[k++];
Long jobID = jsr.getJobID();
IJobDescription job = jobManager.load(jobID,true);
if (job != null)
@@ -170,7 +170,7 @@ public class JobNotificationThread exten
int i = 0;
while (i < jobsNeedingNotification.length)
{
- JobStartRecord jsr = jobsNeedingNotification[i++];
+ JobNotifyRecord jsr = jobsNeedingNotification[i++];
if (!jsr.wasStarted())
{
// Clean up from failed start.
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java Sat Mar 16 21:03:08 2013
@@ -3118,13 +3118,13 @@ public class ManifoldCF extends org.apac
/** Start a job.
*/
- protected static int apiWriteStartJob(IThreadContext tc, Configuration output, Long jobID)
+ protected static int apiWriteStartJob(IThreadContext tc, Configuration output, Long jobID, boolean requestMinimum)
throws ManifoldCFException
{
try
{
IJobManager jobManager = JobManagerFactory.make(tc);
- jobManager.manualStart(jobID);
+ jobManager.manualStart(jobID,requestMinimum);
return WRITERESULT_CREATED;
}
catch (ManifoldCFException e)
@@ -3154,13 +3154,13 @@ public class ManifoldCF extends org.apac
/** Restart a job.
*/
- protected static int apiWriteRestartJob(IThreadContext tc, Configuration output, Long jobID)
+ protected static int apiWriteRestartJob(IThreadContext tc, Configuration output, Long jobID, boolean requestMinimum)
throws ManifoldCFException
{
try
{
IJobManager jobManager = JobManagerFactory.make(tc);
- jobManager.manualAbortRestart(jobID);
+ jobManager.manualAbortRestart(jobID,requestMinimum);
return WRITERESULT_CREATED;
}
catch (ManifoldCFException e)
@@ -3379,7 +3379,12 @@ public class ManifoldCF extends org.apac
if (path.startsWith("start/"))
{
Long jobID = new Long(path.substring("start/".length()));
- return apiWriteStartJob(tc,output,jobID);
+ return apiWriteStartJob(tc,output,jobID,false);
+ }
+ else if (path.startsWith("startminimal/"))
+ {
+ Long jobID = new Long(path.substring("startminimal/".length()));
+ return apiWriteStartJob(tc,output,jobID,true);
}
else if (path.startsWith("abort/"))
{
@@ -3389,7 +3394,12 @@ public class ManifoldCF extends org.apac
else if (path.startsWith("restart/"))
{
Long jobID = new Long(path.substring("restart/".length()));
- return apiWriteRestartJob(tc,output,jobID);
+ return apiWriteRestartJob(tc,output,jobID,false);
+ }
+ else if (path.startsWith("restartminimal/"))
+ {
+ Long jobID = new Long(path.substring("restartminimal/".length()));
+ return apiWriteRestartJob(tc,output,jobID,true);
}
else if (path.startsWith("pause/"))
{
@@ -3580,6 +3590,7 @@ public class ManifoldCF extends org.apac
protected static final String JOBNODE_SCHEDULE = "schedule";
protected static final String JOBNODE_LINKTYPE = "link_type";
protected static final String JOBNODE_COUNT = "count";
+ protected static final String JOBNODE_REQUESTMINIMUM = "requestminimum";
protected static final String JOBNODE_TIMEZONE = "timezone";
protected static final String JOBNODE_DURATION = "duration";
protected static final String JOBNODE_DAYOFWEEK = "dayofweek";
@@ -3732,6 +3743,7 @@ public class ManifoldCF extends org.apac
// Create a schedule record.
String timezone = null;
Long duration = null;
+ boolean requestMinimum = false;
EnumeratedValues dayOfWeek = null;
EnumeratedValues monthOfYear = null;
EnumeratedValues dayOfMonth = null;
@@ -3745,6 +3757,10 @@ public class ManifoldCF extends org.apac
{
ConfigurationNode scheduleField = child.findChild(q++);
String fieldType = scheduleField.getType();
+ if (fieldType.equals(JOBNODE_REQUESTMINIMUM))
+ {
+ requestMinimum = scheduleField.getValue().equals("true");
+ }
if (fieldType.equals(JOBNODE_TIMEZONE))
{
timezone = scheduleField.getValue();
@@ -3780,7 +3796,7 @@ public class ManifoldCF extends org.apac
else
throw new ManifoldCFException("Unrecognized field in schedule record: '"+fieldType+"'");
}
- ScheduleRecord sr = new ScheduleRecord(dayOfWeek,monthOfYear,dayOfMonth,year,hourOfDay,minutesOfHour,timezone,duration);
+ ScheduleRecord sr = new ScheduleRecord(dayOfWeek,monthOfYear,dayOfMonth,year,hourOfDay,minutesOfHour,timezone,duration,requestMinimum);
// Add the schedule record to the job.
jobDescription.addScheduleRecord(sr);
}
@@ -3931,6 +3947,11 @@ public class ManifoldCF extends org.apac
child = new ConfigurationNode(JOBNODE_SCHEDULE);
ConfigurationNode recordChild;
+ // requestminimum
+ recordChild = new ConfigurationNode(JOBNODE_REQUESTMINIMUM);
+ recordChild.setValue(sr.getRequestMinimum()?"true":"false");
+ child.addChild(child.getChildCount(),recordChild);
+
// timezone
if (sr.getTimezone() != null)
{
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java Sat Mar 16 21:03:08 2013
@@ -91,7 +91,7 @@ public class SeedingThread extends Threa
Logging.threads.debug("Seeding thread woke up");
// Grab active, adaptive jobs (and set their state to xxxSEEDING as a side effect)
- JobStartRecord[] seedJobs = jobManager.getJobsReadyForSeeding(currentTime);
+ JobSeedingRecord[] seedJobs = jobManager.getJobsReadyForSeeding(currentTime);
// Process these jobs, and do the seeding. The seeding is based on what came back
// in the job start record for sync time. If there's an interruption, we just go on
@@ -113,7 +113,7 @@ public class SeedingThread extends Threa
int i = 0;
while (i < seedJobs.length)
{
- JobStartRecord jsr = seedJobs[i++];
+ JobSeedingRecord jsr = seedJobs[i++];
Long jobID = jsr.getJobID();
try
{
@@ -203,7 +203,7 @@ public class SeedingThread extends Threa
int i = 0;
while (i < seedJobs.length)
{
- JobStartRecord jsr = seedJobs[i++];
+ JobSeedingRecord jsr = seedJobs[i++];
if (!jsr.wasStarted())
{
if (Logging.threads.isDebugEnabled())
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java Sat Mar 16 21:03:08 2013
@@ -80,7 +80,7 @@ public class StartDeleteThread extends T
// See if there are any starting jobs.
// Note: Since this following call changes the job state, we must be careful to reset it on any kind of failure.
- JobStartRecord[] deleteJobs = jobManager.getJobsReadyForDelete();
+ JobDeleteRecord[] deleteJobs = jobManager.getJobsReadyForDelete();
try
{
@@ -100,7 +100,7 @@ public class StartDeleteThread extends T
int i = 0;
while (i < deleteJobs.length)
{
- JobStartRecord jsr = deleteJobs[i++];
+ JobDeleteRecord jsr = deleteJobs[i++];
Long jobID = jsr.getJobID();
try
{
@@ -128,7 +128,7 @@ public class StartDeleteThread extends T
int i = 0;
while (i < deleteJobs.length)
{
- JobStartRecord jsr = deleteJobs[i++];
+ JobDeleteRecord jsr = deleteJobs[i++];
if (!jsr.wasStarted())
{
// Clean up from failed start.
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java Sat Mar 16 21:03:08 2013
@@ -135,38 +135,15 @@ public class StartupThread extends Threa
// Get the number of link types.
String[] legalLinkTypes = connector.getRelationshipTypes();
- // The old logic here looked at the model, and if it was incomplete, either
- // performed a complete crawl initialization, or an incremental crawl initialization.
- // The fact is that we can now determine automatically what kind of crawl should be
- // done, based on the model the connector states and on the starting time that we
- // would be feeding it. (The starting time is reset to 0 when the document specification
- // is changed - that's a crucial consideration.)
- //
- // The new logic does this:
- //
- // (1) If the connector has MODEL_ADD_CHANGE_DELETE, then
- // we let the connector run the show; there's no purge phase, and therefore the
- // documents are left in a COMPLETED state if they don't show up in the list
- // of seeds that require the attention of the connector.
- //
- // (2) If the connector has MODEL_ALL, then it's a full crawl no matter what, so
- // we do a full scan initialization.
- //
- // (3) If the connector has some other model, we look at the start time. A start
- // time of 0 implies a full scan, while any other start time implies an incremental
- // scan.
-
- if (model != connector.MODEL_ADD_CHANGE_DELETE)
- {
- if (Logging.threads.isDebugEnabled())
- Logging.threads.debug("Preparing job "+jobID.toString()+" for execution...");
- if (jobType != IJobDescription.TYPE_CONTINUOUS && model != connector.MODEL_PARTIAL && (model == connector.MODEL_ALL || lastJobTime == 0L))
- jobManager.prepareFullScan(jobID,legalLinkTypes,hopcountMethod);
- else
- jobManager.prepareIncrementalScan(jobID,legalLinkTypes,hopcountMethod);
- if (Logging.threads.isDebugEnabled())
- Logging.threads.debug("Prepared job "+jobID.toString()+" for execution.");
- }
+ boolean requestMinimum = jsr.getRequestMinimum();
+
+ if (Logging.threads.isDebugEnabled())
+ Logging.threads.debug("Preparing job "+jobID.toString()+" for execution...");
+ jobManager.prepareJobScan(jobID,legalLinkTypes,hopcountMethod,
+ model,jobType == IJobDescription.TYPE_CONTINUOUS,lastJobTime == 0L,
+ requestMinimum);
+ if (Logging.threads.isDebugEnabled())
+ Logging.threads.debug("Prepared job "+jobID.toString()+" for execution.");
try
{
Modified: manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties (original)
+++ manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties Sat Mar 16 21:03:08 2013
@@ -34,10 +34,6 @@ editjob.Scheduling=Scheduling
editjob.HopFilters=Hop Filters
editjob.am=am
editjob.pm=pm
-editjob.st=st
-editjob.nd=nd
-editjob.rd=rd
-editjob.th=th
index.WelcomeToApacheManifoldFC=Welcome to Apache ManifoldCF
@@ -341,6 +337,14 @@ editjob.NoForcedMetadataSpecified=No for
editjob.Add=Add
editjob.Addforcedmetadata=Add forced metadata
editjob.ForcedMetadataNameMustNotBeNull=Forced metadata name must not be null
+editjob.st=st
+editjob.nd=nd
+editjob.rd=rd
+editjob.th=th
+editjob.dayofmonth=day of month
+editjob.JobInvocationColon=Job invocation:
+editjob.Minimal=Minimal
+editjob.Complete=Complete
editjob.tab=tab
@@ -357,6 +361,39 @@ showjobstatus.Refresh=Refresh
showjobstatus.PleaseTryAgainLater=This page is unavailable due to maintenance operations. Please try again later.
showjobstatus.StatusOfJobs=Status of Jobs
+showjobstatus.Notyetrun=Not yet run
+showjobstatus.Running=Running
+showjobstatus.Runningnoconnector=Running, no connector
+showjobstatus.Aborting=Aborting
+showjobstatus.Restarting=Restarting
+showjobstatus.Stopping=Stopping
+showjobstatus.Resuming=Resuming
+showjobstatus.Paused=Paused
+showjobstatus.Done=Done
+showjobstatus.Waiting=Waiting
+showjobstatus.Startingup=Starting up
+showjobstatus.Cleaningup=Cleaning up
+showjobstatus.Terminating=Terminating
+showjobstatus.Endnotification=End notification
+showjobstatus.ErrorColon=Error:
+showjobstatus.Unknown=Unknown
+showjobstatus.Notstarted=Not started
+showjobstatus.Aborted=Aborted
+showjobstatus.Neverrun=Never run
+showjobstatus.Start=Start
+showjobstatus.Startminimal=Start minimal
+showjobstatus.Restart=Restart
+showjobstatus.Restartminimal=Restart minimal
+showjobstatus.Pause=Pause
+showjobstatus.Abort=Abort
+showjobstatus.Resume=Resume
+showjobstatus.Startjob=Start job
+showjobstatus.minimally=minimally
+showjobstatus.Restartjob=Restart job
+showjobstatus.Pausejob=Pause job
+showjobstatus.Abortjob=Abort job
+showjobstatus.Resumejob=Resume job
+
documentstatus.ApacheManifoldCFDocumentStatus=Apache ManifoldCF: Document Status
documentstatus.DocumentStatus=Document Status
documentstatus.Connection=Connection:
@@ -686,7 +723,7 @@ viewjob.at=at
viewjob.am=am
viewjob.pm=pm
viewjob.plus=plus
-viewjob.inJanuary=in January
+viewjob.ineverymonthofyear=in every month of the year
viewjob.in=in
viewjob.January=January
viewjob.February=February
@@ -713,6 +750,9 @@ viewjob.st=st
viewjob.nd=nd
viewjob.rd=rd
viewjob.th=th
+viewjob.JobInvocationColon=Job invocation:
+viewjob.Minimal=Minimal
+viewjob.Complete=Complete
viewauthority.ViewAuthorityConnectionStatus=View Authority Connection Status
viewauthority.NameColon=Name:
Modified: manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties (original)
+++ manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties Sat Mar 16 21:03:08 2013
@@ -34,10 +34,6 @@ editjob.Scheduling=ã¹ã±ã¸ã�
editjob.HopFilters=ããããã£ã«ã¿
editjob.am=åå
editjob.pm=åå¾
-editjob.st=æ¥
-editjob.nd=æ¥
-editjob.rd=æ¥
-editjob.th=æ¥
index.WelcomeToApacheManifoldFC=Apache ManifoldCFã¸ãããã
@@ -341,6 +337,14 @@ editjob.NoForcedMetadataSpecified=No for
editjob.Add=Add
editjob.Addforcedmetadata=Add forced metadata
editjob.ForcedMetadataNameMustNotBeNull=Forced metadata name must not be null
+editjob.st=æ¥
+editjob.nd=æ¥
+editjob.rd=æ¥
+editjob.th=æ¥
+editjob.dayofmonth=day of month
+editjob.JobInvocationColon=Job invocation:
+editjob.Minimal=Minimal
+editjob.Complete=Complete
editjob.tab=tab
@@ -357,6 +361,39 @@ showjobstatus.Refresh=æ´æ°
showjobstatus.PleaseTryAgainLater=ä¿å®å¦çä¸ã§ããå°ã
ãå¾
ã¡ãã ããã
showjobstatus.StatusOfJobs=ã¸ã§ãã®ç¶æ
+showjobstatus.Notyetrun=Not yet run
+showjobstatus.Running=Running
+showjobstatus.Runningnoconnector=Running, no connector
+showjobstatus.Aborting=Aborting
+showjobstatus.Restarting=Restarting
+showjobstatus.Stopping=Stopping
+showjobstatus.Resuming=Resuming
+showjobstatus.Paused=Paused
+showjobstatus.Done=Done
+showjobstatus.Waiting=Waiting
+showjobstatus.Startingup=Starting up
+showjobstatus.Cleaningup=Cleaning up
+showjobstatus.Terminating=Terminating
+showjobstatus.Endnotification=End notification
+showjobstatus.ErrorColon=Error:
+showjobstatus.Unknown=Unknown
+showjobstatus.Notstarted=Not started
+showjobstatus.Aborted=Aborted
+showjobstatus.Neverrun=Never run
+showjobstatus.Start=Start
+showjobstatus.Startminimal=Start minimal
+showjobstatus.Restart=Restart
+showjobstatus.Restartminimal=Restart minimal
+showjobstatus.Pause=Pause
+showjobstatus.Abort=Abort
+showjobstatus.Resume=Resume
+showjobstatus.Startjob=Start job
+showjobstatus.minimally=minimally
+showjobstatus.Restartjob=Restart job
+showjobstatus.Pausejob=Pause job
+showjobstatus.Abortjob=Abort job
+showjobstatus.Resumejob=Resume job
+
documentstatus.ApacheManifoldCFDocumentStatus=Apache ManifoldCFï¼ã³ã³ãã³ãã®ç¶æ
documentstatus.DocumentStatus=ã³ã³ãã³ãã®ç¶æ
documentstatus.Connection=ã³ãã¯ã·ã§ã³ï¼
@@ -673,34 +710,34 @@ viewjob.Scaneverydocumentonce=Scan every
viewjob.Startatbeginningofschedulewindow=Start at beginning of schedule window
viewjob.Startinsideschedulewindow=Start inside schedule window
viewjob.Dontautomaticallystart=Don't automatically start
-viewjob.Anydayoftheweek=Any day of week
-viewjob.Sundays=Sundays
-viewjob.Mondays=Mondays
-viewjob.Tuesdays=Tuesdays
-viewjob.Wednesdays=Wednesdays
-viewjob.Thursdays=Thursdays
-viewjob.Fridays=Fridays
-viewjob.Saturdays=Saturdays
+viewjob.Anydayoftheweek=ãã¹ã¦ã®ææ¥
+viewjob.Sundays=æ¥
+viewjob.Mondays=æ
+viewjob.Tuesdays=ç«
+viewjob.Wednesdays=æ°´
+viewjob.Thursdays=æ¨
+viewjob.Fridays=é
+viewjob.Saturdays=å
viewjob.oneveryhour=on every hour
viewjob.atmidnight=at midnight
viewjob.at=at
-viewjob.am=am
-viewjob.pm=pm
+viewjob.am=åå
+viewjob.pm=åå¾
viewjob.plus=plus
-viewjob.inJanuary=in January
-viewjob.in=in
-viewjob.January=January
-viewjob.February=February
-viewjob.March=March
-viewjob.April=April
-viewjob.May=May
-viewjob.June=June
-viewjob.July=July
-viewjob.August=August
-viewjob.September=September
-viewjob.October=October
-viewjob.November=November
-viewjob.December=December
+viewjob.ineverymonthofyear=ãã¹ã¦ã®æ
+viewjob.in=
+viewjob.January=ï¼æ
+viewjob.February=ï¼æ
+viewjob.March=ï¼æ
+viewjob.April=ï¼æ
+viewjob.May=ï¼æ
+viewjob.June=ï¼æ
+viewjob.July=ï¼æ
+viewjob.August=ï¼æ
+viewjob.September=ï¼æ
+viewjob.October=ï¼ï¼æ
+viewjob.November=ï¼ï¼æ
+viewjob.December=ï¼ï¼æ
viewjob.onthe1stofthemonth=on the 1st of the month
viewjob.onthe=on the
viewjob.ofthemonth=of the month
@@ -710,10 +747,13 @@ viewjob.Unlimited=Unlimited
viewjob.Deleteunreachabledocuments=Delete unreachable documents
viewjob.Nodeletesfornow=No deletes, for now
viewjob.Nodeletesforever=No deletes, forever
-viewjob.st=st
-viewjob.nd=nd
-viewjob.rd=rd
-viewjob.th=th
+viewjob.st=æ¥
+viewjob.nd=æ¥
+viewjob.rd=æ¥
+viewjob.th=æ¥
+viewjob.JobInvocationColon=Job invocation:
+viewjob.Minimal=Minimal
+viewjob.Complete=Complete
viewauthority.ViewAuthorityConnectionStatus=権éã³ãã¯ã·ã§ã³ç¶æ
ã®è¡¨ç¤º
viewauthority.NameColon=ååï¼
Modified: manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/end-user-documentation.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/end-user-documentation.xml?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/end-user-documentation.xml (original)
+++ manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/end-user-documentation.xml Sat Mar 16 21:03:08 2013
@@ -284,15 +284,20 @@
status column. Allowed actions you may see at one point or another include:</p>
<ul>
<li>Start (start the job)</li>
+ <li>Start minimal (start the job, but do only the minimal work possible)</li>
<li>Abort(abort the job)</li>
<li>Pause (pause the job)</li>
<li>Resume (resume the job)</li>
<li>Restart (equivalent to aborting the job, and starting it all over again)</li>
+ <li>Restart minimal (equivalent to aborting the job, and starting it all over again, doing only the minimal work possible)</li>
</ul>
<br/>
<p>The columns "Documents", "Active", and "Processed" have very specific means as far as documents in the job's queue are concerned. The "Documents" column counts all the documents
that belong to the job. The "Active" column counts all of the documents for that job that are queued up for processing. The "Processed" column counts all documents that are on the
queue for the job that have been processed at least once in the past.</p>
+ <p>Using the "minimal" variant of the listed actions will perform the minimum possible amount of work, given the model that the connection type for the job uses. In some
+ cases, this will mean that additions and modifications are indexed, but deletions are not detected. A complete job run is usually necessary to fully synchronize the target
+ index with the repository contents.</p>
</section>
<section id="statusreports">
<title>Status Reports</title>