You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/01/05 18:08:28 UTC
svn commit: r1555583 - in /manifoldcf/trunk: ./
framework/core/src/test/resources/org/apache/manifoldcf/core/tests/
framework/crawler-ui/src/main/webapp/
framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/
framework/pull-agent/...
Author: kwright
Date: Sun Jan 5 17:08:27 2014
New Revision: 1555583
URL: http://svn.apache.org/r1555583
Log:
Fix for CONNECTORS-850. Add maxinterval field to jobs. Warning: Schema change!!
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py
manifoldcf/trunk/framework/crawler-ui/src/main/webapp/editjob.jsp
manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp
manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewjob.jsp
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobDescription.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobDescription.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties
Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1555583&r1=1555582&r2=1555583&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Sun Jan 5 17:08:27 2014
@@ -3,6 +3,10 @@ $Id$
======================= 1.5-dev =====================
+CONNECTORS-850: Add maximum recrawl interval parameter for all
+jobs. NOTE: Schema change!!
+(Florian Schmedding, Karl Wright)
+
CONNECTORS-849: Upgrade to Hadoop 2.2.0.
(Karl Wright)
Modified: manifoldcf/trunk/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py?rev=1555583&r1=1555582&r2=1555583&view=diff
==============================================================================
--- manifoldcf/trunk/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py (original)
+++ manifoldcf/trunk/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py Sun Jan 5 17:08:27 2014
@@ -805,9 +805,12 @@ class VirtualWindow:
# confirm( )
# alert( )
# eval( )
+ # parseInt( )
+
self.jscontext.define_value( "confirm", JSConfirmMethod( self ) )
self.jscontext.define_value( "alert", JSAlertMethod( self ))
self.jscontext.define_value( "eval", JSEvalMethod( self ))
+ self.jscontext.define_value( "parseInt", JSParseIntMethod( self ))
# Need built-in objects for:
# document (with form properties and form element properties beneath that).
@@ -1368,6 +1371,22 @@ class JSEvalMethod( Javascript.JSObject
tokenstream = Javascript.JSTokenStream( message )
return tokenstream.evaluate_expr( context, "Eval" )
+# JS object that handles "parseInt" method
+class JSParseIntMethod( Javascript.JSObject ):
+
+ def __init__( self, window_instance ):
+ Javascript.JSObject.__init__( self )
+ assert isinstance( window_instance, VirtualWindow )
+ self.window_instance = window_instance
+
+ def call( self, argset, context ):
+ # Check to be sure we have one string argument
+ if len(argset) != 1:
+ raise Exception("parseInt method requires one string argument")
+ # Evaluate to int
+ intvalue = argset[0].int_value( )
+ return Javascript.JSNumber( intvalue )
+
# JS object that handles "submit" method
class JSSubmitMethod( Javascript.JSObject ):
Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/editjob.jsp
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/editjob.jsp?rev=1555583&r1=1555582&r2=1555583&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/editjob.jsp (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/editjob.jsp Sun Jan 5 17:08:27 2014
@@ -85,6 +85,8 @@
int priority = 5;
// Minimum recrawl interval (Default: 1 day)
Long recrawlInterval = new Long(60L * 24L);
+ // Maximum recrawl interval (Default: none)
+ Long maxRecrawlInterval = null;
// Reseed interval (Default: 60 minutes)
Long reseedInterval = new Long(60L);
// Expiration interval (Default: never)
@@ -120,6 +122,8 @@
priority = job.getPriority();
Long value = job.getInterval();
recrawlInterval = (value==null)?null:new Long(value.longValue()/60000L);
+ value = job.getMaxInterval();
+ maxRecrawlInterval = (value==null)?null:new Long(value.longValue()/60000L);
value = job.getReseedInterval();
reseedInterval = (value==null)?null:new Long(value.longValue()/60000L);
value = job.getExpiration();
@@ -310,6 +314,10 @@
{
if (!checkRecrawl())
return false;
+ if (!checkMaxRecrawl())
+ return false;
+ if (!checkRecrawlConsistent())
+ return false;
if (!checkReseed())
return false;
if (!checkExpiration())
@@ -360,6 +368,28 @@
return true;
}
+ function checkMaxRecrawl()
+ {
+ if (editjob.maxrecrawlinterval.value != "" && !isInteger(editjob.maxrecrawlinterval.value))
+ {
+ alert("<%=Messages.getBodyJavascriptString(pageContext.getRequest().getLocale(),"editjob.MaxRecrawlIntervalMustBeAValidIntegerOrNull")%>");
+ editjob.maxrecrawlinterval.focus();
+ return false;
+ }
+ return true;
+ }
+
+ function checkRecrawlConsistent()
+ {
+ if (editjob.maxrecrawlinterval.value != "" && editjob.recrawlinterval.value != "" && parseInt(editjob.maxrecrawlinterval.value) < parseInt(editjob.recrawlinterval.value))
+ {
+ alert("<%=Messages.getBodyJavascriptString(pageContext.getRequest().getLocale(),"editjob.MaxRecrawlIntervalMustBeLargerThanRecrawlInterval")%>");
+ editjob.maxrecrawlinterval.focus();
+ return false;
+ }
+ return true;
+ }
+
function checkReseed()
{
if (editjob.reseedinterval.value != "" && !isInteger(editjob.reseedinterval.value))
@@ -843,18 +873,26 @@
<td class="description">
<nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.ScheduleTypeColon")%></nobr>
</td>
- <td class="value">
+ <td class="value" colspan="3">
<select name="scheduletype" size="1">
<option value='<%=IJobDescription.TYPE_CONTINUOUS%>' <%=(type==IJobDescription.TYPE_CONTINUOUS)?"selected=\"selected\"":""%>><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.RescanDocumentsDynamically")%></option>
<option value='<%=IJobDescription.TYPE_SPECIFIED%>' <%=(type==IJobDescription.TYPE_SPECIFIED)?"selected=\"selected\"":""%>><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.ScanEveryDocumentOnce")%></option>
</select>
</td>
+ </tr>
+ <tr>
<td class="description">
<nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.RecrawlIntervalIfContinuousColon")%></nobr>
</td>
<td class="value">
<nobr><input type="text" size="5" name="recrawlinterval" value='<%=((recrawlInterval==null)?"":recrawlInterval.toString())%>'/> <%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.minutesBlankInfinity")%></nobr>
</td>
+ <td class="description">
+ <nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.MaxRecrawlIntervalIfContinuousColon")%></nobr>
+ </td>
+ <td class="value">
+ <nobr><input type="text" size="5" name="maxrecrawlinterval" value='<%=((maxRecrawlInterval==null)?"":maxRecrawlInterval.toString())%>'/> <%=Messages.getBodyString(pageContext.getRequest().getLocale(),"editjob.minutesBlankInfinity")%></nobr>
+ </td>
</tr>
<tr>
<td class="description">
@@ -877,6 +915,7 @@
%>
<input type="hidden" name="scheduletype" value='<%=type%>'/>
<input type="hidden" name="recrawlinterval" value='<%=((recrawlInterval==null)?"":recrawlInterval.toString())%>'/>
+ <input type="hidden" name="maxrecrawlinterval" value='<%=((maxRecrawlInterval==null)?"":maxRecrawlInterval.toString())%>'/>
<input type="hidden" name="reseedinterval" value='<%=((reseedInterval==null)?"":reseedInterval.toString())%>'/>
<input type="hidden" name="expirationinterval" value='<%=((expirationInterval==null)?"":expirationInterval.toString())%>'/>
<%
@@ -1155,6 +1194,7 @@
%>
<input type="hidden" name="scheduletype" value='<%=type%>'/>
<input type="hidden" name="recrawlinterval" value='<%=((recrawlInterval==null)?"":recrawlInterval.toString())%>'/>
+ <input type="hidden" name="maxrecrawlinterval" value='<%=((maxRecrawlInterval==null)?"":maxRecrawlInterval.toString())%>'/>
<input type="hidden" name="reseedinterval" value='<%=((reseedInterval==null)?"":reseedInterval.toString())%>'/>
<input type="hidden" name="expirationinterval" value='<%=((expirationInterval==null)?"":expirationInterval.toString())%>'/>
<%
Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp?rev=1555583&r1=1555582&r2=1555583&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/execute.jsp Sun Jan 5 17:08:27 2014
@@ -1005,6 +1005,14 @@
else
job.setInterval(new Long(new Long(x).longValue() * 60000L));
}
+ x = variableContext.getParameter("maxrecrawlinterval");
+ if (x != null)
+ {
+ if (x.length() == 0)
+ job.setMaxInterval(null);
+ else
+ job.setMaxInterval(new Long(new Long(x).longValue() * 60000L));
+ }
x = variableContext.getParameter("reseedinterval");
if (x != null)
{
Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewjob.jsp
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewjob.jsp?rev=1555583&r1=1555582&r2=1555583&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewjob.jsp (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/viewjob.jsp Sun Jan 5 17:08:27 2014
@@ -85,6 +85,7 @@
String naMessage = Messages.getString(pageContext.getRequest().getLocale(),"viewjob.Notapplicable");
String jobType = "";
String intervalString = naMessage;
+ String maxIntervalString = naMessage;
String reseedIntervalString = naMessage;
String expirationIntervalString = naMessage;
@@ -95,9 +96,11 @@
String minutesMessage = Messages.getString(pageContext.getRequest().getLocale(),"viewjob.minutes");
jobType = Messages.getString(pageContext.getRequest().getLocale(),"viewjob.Rescandocumentsdynamically");
Long recrawlInterval = job.getInterval();
+ Long maxRecrawlInterval = job.getMaxInterval();
Long reseedInterval = job.getReseedInterval();
Long expirationInterval = job.getExpiration();
intervalString = (recrawlInterval==null)?infinityMessage:(new Long(recrawlInterval.longValue()/60000L).toString()+" "+minutesMessage);
+ maxIntervalString = (maxRecrawlInterval==null)?infinityMessage:(new Long(maxRecrawlInterval.longValue()/60000L).toString()+" "+minutesMessage);
reseedIntervalString = (reseedInterval==null)?infinityMessage:(new Long(reseedInterval.longValue()/60000L).toString()+" "+minutesMessage);
expirationIntervalString = (expirationInterval==null)?infinityMessage:(new Long(expirationInterval.longValue()/60000L).toString()+" "+minutesMessage);
break;
@@ -173,9 +176,13 @@
</tr>
<tr>
<td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.ScheduleTypeColon")%></nobr></td>
- <td class="value"><nobr><%=jobType%></nobr></td>
+ <td class="value" colspan="3"><nobr><%=jobType%></nobr></td>
+ </tr>
+ <tr>
<td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.MinimumRecrawlIntervalColon")%></nobr></td>
<td class="value"><nobr><%=intervalString%></nobr></td>
+ <td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.MaximumRecrawlIntervalColon")%></nobr></td>
+ <td class="value"><nobr><%=maxIntervalString%></nobr></td>
</tr>
<tr>
<td class="description"><nobr><%=Messages.getBodyString(pageContext.getRequest().getLocale(),"viewjob.ExpirationIntervalColon")%></nobr></td>
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobDescription.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobDescription.java?rev=1555583&r1=1555582&r2=1555583&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobDescription.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobDescription.java Sun Jan 5 17:08:27 2014
@@ -154,6 +154,16 @@ public interface IJobDescription
*/
public Long getInterval();
+ /** Set the maximum rescheduling interval, in milliseconds, or null if forever.
+ *@param interval is the maximum interval.
+ */
+ public void setMaxInterval(Long interval);
+
+ /** Get the maximum rescheduling interval, in milliseconds.
+ *@return the max interval, or null if forever.
+ */
+ public Long getMaxInterval();
+
/** Set the expiration time, in milliseconds.
*@param time is the maximum expiration time of a document, in milliseconds, or null if none.
*/
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobDescription.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobDescription.java?rev=1555583&r1=1555582&r2=1555583&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobDescription.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobDescription.java Sun Jan 5 17:08:27 2014
@@ -57,6 +57,9 @@ public class JobDescription implements I
// Default interval for continuous crawling
protected Long interval = new Long(1000L*3600L*24L); // 1 day is the default
+ // Maximum interval for continuous crawling
+ protected Long maxInterval = null;
+
// Document expiration time for this job, in milliseconds
protected Long expiration = null; // Never is the default
@@ -99,6 +102,7 @@ public class JobDescription implements I
// No direct modification of this object is possible
rval.scheduleList = scheduleList.duplicate();
rval.interval = interval;
+ rval.maxInterval = maxInterval;
rval.expiration = expiration;
rval.reseedInterval = reseedInterval;
rval.rate = rate;
@@ -153,6 +157,7 @@ public class JobDescription implements I
/** Get isnew.
*@return true if the object is new.
*/
+ @Override
public boolean getIsNew()
{
return isNew;
@@ -171,6 +176,7 @@ public class JobDescription implements I
/** Get the id.
*@return the id.
*/
+ @Override
public Long getID()
{
return id;
@@ -179,6 +185,7 @@ public class JobDescription implements I
/** Set the description.
*@param description is the description.
*/
+ @Override
public void setDescription(String description)
{
if (readOnly)
@@ -189,6 +196,7 @@ public class JobDescription implements I
/** Get the description.
*@return the description
*/
+ @Override
public String getDescription()
{
return description;
@@ -197,6 +205,7 @@ public class JobDescription implements I
/** Set the output connection name.
*@param connectionName is the connection name.
*/
+ @Override
public void setOutputConnectionName(String connectionName)
{
if (readOnly)
@@ -207,6 +216,7 @@ public class JobDescription implements I
/** Get the output connection name.
*@return the output connection name.
*/
+ @Override
public String getOutputConnectionName()
{
return outputConnectionName;
@@ -215,6 +225,7 @@ public class JobDescription implements I
/** Set the connection name.
*@param connectionName is the connection name.
*/
+ @Override
public void setConnectionName(String connectionName)
{
if (readOnly)
@@ -225,6 +236,7 @@ public class JobDescription implements I
/** Get the connection name.
*@return the connection name.
*/
+ @Override
public String getConnectionName()
{
return connectionName;
@@ -233,6 +245,7 @@ public class JobDescription implements I
/** Set the job type.
*@param type is the type (as an integer).
*/
+ @Override
public void setType(int type)
{
if (readOnly)
@@ -243,6 +256,7 @@ public class JobDescription implements I
/** Get the job type.
*@return the type (as an integer).
*/
+ @Override
public int getType()
{
return type;
@@ -251,6 +265,7 @@ public class JobDescription implements I
/** Set the job's start method.
*@param startMethod is the start description.
*/
+ @Override
public void setStartMethod(int startMethod)
{
if (readOnly)
@@ -261,6 +276,7 @@ public class JobDescription implements I
/** Get the job's start method.
*@return the start method.
*/
+ @Override
public int getStartMethod()
{
return startMethod;
@@ -272,6 +288,7 @@ public class JobDescription implements I
/** Clear all the scheduling records.
*/
+ @Override
public void clearScheduleRecords()
{
if (readOnly)
@@ -282,6 +299,7 @@ public class JobDescription implements I
/** Add a record.
*@param record is the record to add.
*/
+ @Override
public void addScheduleRecord(ScheduleRecord record)
{
if (readOnly)
@@ -292,6 +310,7 @@ public class JobDescription implements I
/** Get the number of schedule records.
*@return the count.
*/
+ @Override
public int getScheduleRecordCount()
{
return scheduleList.getRecordCount();
@@ -301,6 +320,7 @@ public class JobDescription implements I
*@param index is the record number.
*@return the record.
*/
+ @Override
public ScheduleRecord getScheduleRecord(int index)
{
return scheduleList.getRecord(index);
@@ -309,6 +329,7 @@ public class JobDescription implements I
/** Delete a specified schedule record.
*@param index is the record number.
*/
+ @Override
public void deleteScheduleRecord(int index)
{
if (readOnly)
@@ -323,6 +344,7 @@ public class JobDescription implements I
/** Set the rescheduling interval, in milliseconds.
*@param interval is the default interval, or null for infinite.
*/
+ @Override
public void setInterval(Long interval)
{
if (readOnly)
@@ -333,14 +355,36 @@ public class JobDescription implements I
/** Get the rescheduling interval, in milliseconds.
*@return the default interval, or null for infinite.
*/
+ @Override
public Long getInterval()
{
return interval;
}
+ /** Set the maximum rescheduling interval, in milliseconds, or null if forever.
+ *@param interval is the maximum interval.
+ */
+ @Override
+ public void setMaxInterval(Long interval)
+ {
+ if (readOnly)
+ throw new IllegalStateException("Attempt to change read-only object");
+ this.maxInterval = interval;
+ }
+
+ /** Get the maximum rescheduling interval, in milliseconds.
+ *@return the max interval, or null if forever.
+ */
+ @Override
+ public Long getMaxInterval()
+ {
+ return maxInterval;
+ }
+
/** Set the expiration time, in milliseconds.
*@param time is the maximum expiration time of a document, in milliseconds, or null if none.
*/
+ @Override
public void setExpiration(Long time)
{
if (readOnly)
@@ -351,6 +395,7 @@ public class JobDescription implements I
/** Get the expiration time, in milliseconds.
*@return the maximum expiration time of a document, or null if none.
*/
+ @Override
public Long getExpiration()
{
return expiration;
@@ -359,6 +404,7 @@ public class JobDescription implements I
/** Set the reseeding interval, in milliseconds.
*@param interval is the interval, or null for infinite.
*/
+ @Override
public void setReseedInterval(Long interval)
{
if (readOnly)
@@ -369,6 +415,7 @@ public class JobDescription implements I
/** Get the reseeding interval, in milliseconds.
*@return the interval, or null if infinite.
*/
+ @Override
public Long getReseedInterval()
{
return reseedInterval;
@@ -377,6 +424,7 @@ public class JobDescription implements I
/** Get the output specification.
*@return the output specification object.
*/
+ @Override
public OutputSpecification getOutputSpecification()
{
return outputSpecification;
@@ -385,6 +433,7 @@ public class JobDescription implements I
/** Get the document specification.
*@return the document specification object.
*/
+ @Override
public DocumentSpecification getSpecification()
{
return documentSpecification;
@@ -395,6 +444,7 @@ public class JobDescription implements I
* 1 is the highest priority.
*@param priority is the priority.
*/
+ @Override
public void setPriority(int priority)
{
if (readOnly)
@@ -405,6 +455,7 @@ public class JobDescription implements I
/** Get the job priority.
*@return the priority (a number between 1 and 10).
*/
+ @Override
public int getPriority()
{
return priority;
@@ -415,6 +466,7 @@ public class JobDescription implements I
/** Get the set of hopcount filters the job has defined.
*@return the set as a map, keyed by Strings and containing Longs.
*/
+ @Override
public Map getHopCountFilters()
{
return (Map)hopCountFilters.clone();
@@ -422,6 +474,7 @@ public class JobDescription implements I
/** Clear the set of hopcount filters for the job.
*/
+ @Override
public void clearHopCountFilters()
{
if (readOnly)
@@ -434,6 +487,7 @@ public class JobDescription implements I
*@param linkType is the type of link the filter applies to.
*@param maxHops is the maximum hop count. Use null to remove a filter.
*/
+ @Override
public void addHopCountFilter(String linkType, Long maxHops)
{
if (readOnly)
@@ -442,12 +496,14 @@ public class JobDescription implements I
}
/** Get the hopcount mode. */
+ @Override
public int getHopcountMode()
{
return hopcountMode;
}
/** Set the hopcount mode. */
+ @Override
public void setHopcountMode(int mode)
{
if (readOnly)
@@ -460,6 +516,7 @@ public class JobDescription implements I
/** Get the forced metadata.
*@return the set as a map, keyed by metadata name, with value a set of strings.
*/
+ @Override
public Map<String,Set<String>> getForcedMetadata()
{
return forcedMetadata;
@@ -467,6 +524,7 @@ public class JobDescription implements I
/** Clear forced metadata.
*/
+ @Override
public void clearForcedMetadata()
{
forcedMetadata.clear();
@@ -474,6 +532,7 @@ public class JobDescription implements I
/** Add a forced metadata name/value pair.
*/
+ @Override
public void addForcedMetadataValue(String name, String value)
{
Set<String> rval = forcedMetadata.get(name);
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java?rev=1555583&r1=1555582&r2=1555583&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java Sun Jan 5 17:08:27 2014
@@ -45,6 +45,7 @@ import java.util.*;
* <tr><td>outputname</td><td>VARCHAR(32)</td><td>Reference:outputconnections.connectionname</td></tr>
* <tr><td>type</td><td>CHAR(1)</td><td></td></tr>
* <tr><td>intervaltime</td><td>BIGINT</td><td></td></tr>
+ * <tr><td>maxintervaltime</td><td>BIGINT</td><td></td></tr>
* <tr><td>expirationtime</td><td>BIGINT</td><td></td></tr>
* <tr><td>windowend</td><td>BIGINT</td><td></td></tr>
* <tr><td>priority</td><td>BIGINT</td><td></td></tr>
@@ -153,6 +154,8 @@ public class Jobs extends org.apache.man
public final static String typeField = "type";
/** This is the minimum reschedule interval for a document being crawled adaptively (in ms.) */
public final static String intervalField = "intervaltime";
+ /** This is the maximum reschedule interval for a document being crawled adaptively (in ms.) */
+ public final static String maxIntervalField = "maxintervaltime";
/** This is the expiration time of documents for a given job (in ms.) */
public final static String expirationField = "expirationtime";
/** This is the job's priority vs. other jobs. */
@@ -341,6 +344,7 @@ public class Jobs extends org.apache.man
map.put(this.outputNameField,new ColumnDescription("VARCHAR(32)",false,false,outputTableName,outputNameField,false));
map.put(typeField,new ColumnDescription("CHAR(1)",false,false,null,null,false));
map.put(intervalField,new ColumnDescription("BIGINT",false,true,null,null,false));
+ map.put(maxIntervalField,new ColumnDescription("BIGINT",false,true,null,null,false));
map.put(expirationField,new ColumnDescription("BIGINT",false,true,null,null,false));
map.put(windowEndField,new ColumnDescription("BIGINT",false,true,null,null,false));
map.put(priorityField,new ColumnDescription("BIGINT",false,false,null,null,false));
@@ -361,6 +365,12 @@ public class Jobs extends org.apache.man
insertMap.put(processIDField,new ColumnDescription("VARCHAR(16)",false,true,null,null,false));
performAlter(insertMap,null,null,null);
}
+ if (existing.get(maxIntervalField) == null)
+ {
+ Map insertMap = new HashMap();
+ insertMap.put(processIDField,new ColumnDescription("BIGINT",false,true,null,null,false));
+ performAlter(insertMap,null,null,null);
+ }
}
// Handle related tables
@@ -764,6 +774,7 @@ public class Jobs extends org.apache.man
values.put(typeField,typeToString(jobDescription.getType()));
values.put(startMethodField,startMethodToString(jobDescription.getStartMethod()));
values.put(intervalField,jobDescription.getInterval());
+ values.put(maxIntervalField,jobDescription.getMaxInterval());
values.put(reseedIntervalField,jobDescription.getReseedInterval());
values.put(expirationField,jobDescription.getExpiration());
values.put(priorityField,new Integer(jobDescription.getPriority()));
@@ -3051,6 +3062,7 @@ public class Jobs extends org.apache.man
Object x;
rc.setInterval((Long)row.getValue(intervalField));
+ rc.setMaxInterval((Long)row.getValue(maxIntervalField));
rc.setReseedInterval((Long)row.getValue(reseedIntervalField));
rc.setExpiration((Long)row.getValue(expirationField));
@@ -3065,7 +3077,7 @@ public class Jobs extends org.apache.man
}
catch (NumberFormatException e)
{
- throw new ManifoldCFException("Bad number",e);
+ throw new ManifoldCFException("Bad number: "+e.getMessage(),e);
}
}
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java?rev=1555583&r1=1555582&r2=1555583&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java Sun Jan 5 17:08:27 2014
@@ -1870,7 +1870,13 @@ public class WorkerThread extends Thread
Long recrawlTime = null;
Long recrawlInterval = job.getInterval();
if (recrawlInterval != null)
- recrawlTime = new Long(currentTime + timeAmt + recrawlInterval.longValue());
+ {
+ Long maxInterval = job.getMaxInterval();
+ long actualInterval = recrawlInterval.longValue() + timeAmt;
+ if (maxInterval != null && actualInterval > maxInterval.longValue())
+ actualInterval = maxInterval.longValue();
+ recrawlTime = new Long(currentTime + actualInterval);
+ }
if (Logging.scheduling.isDebugEnabled())
Logging.scheduling.debug("Default rescan time for document '"+localIdentifier+"' is "+((recrawlTime==null)?"NEVER":recrawlTime.toString()));
Long lowerBound = getDocumentRescheduleLowerBoundTime(localIdentifier);
Modified: manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties?rev=1555583&r1=1555582&r2=1555583&view=diff
==============================================================================
--- manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties (original)
+++ manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties Sun Jan 5 17:08:27 2014
@@ -368,6 +368,7 @@ editjob.ScheduleTypeColon=Schedule type:
editjob.RescanDocumentsDynamically=Rescan documents dynamically
editjob.ScanEveryDocumentOnce=Scan every document once
editjob.RecrawlIntervalIfContinuousColon=Recrawl interval (if continuous):
+editjob.MaxRecrawlIntervalIfContinuousColon=Maximum recrawl interval (if continuous):
editjob.minutesBlankInfinity=minutes (blank=infinity)
editjob.ExpirationIntervalIfContinuousColon=Expiration interval (if continuous):
editjob.ReseedIntervalIfContinuousColon=Reseed interval (if continuous):
@@ -415,6 +416,8 @@ editjob.AddNewScheduleRecord=Add new sch
editjob.JobMustHaveAName=Job must have a name
editjob.DurationMustBeAValidInteger=Duration must be a valid integer
editjob.RecrawlIntervalMustBeAValidIntegerOrNull=Recrawl interval must be a valid integer or null
+editjob.MaxRecrawlIntervalMustBeAValidIntegerOrNull=Maximum recrawl interval must be a valid integer or null
+editjob.MaxRecrawlIntervalMustBeLargerThanRecrawlInterval=Maximum recrawl interval must be larger that recrawl interval
editjob.ReseedIntervalMustBeAValidIntegerOrNull=Reseed interval must be a valid integer or null
editjob.ExpirationIntervalMustBeAValidIntegerOrNull=Expiration interval must be a valid integer or null
editjob.RemoveScheduleRecord=Remove schedule record #
@@ -773,6 +776,7 @@ viewjob.ApacheManifoldCFViewJob=Apache M
viewjob.ViewAJob=View a Job
viewjob.ScheduleTypeColon=Schedule type:
viewjob.MinimumRecrawlIntervalColon=Minimum recrawl interval:
+viewjob.MaximumRecrawlIntervalColon=Maximum recrawl interval:
viewjob.ExpirationIntervalColon=Expiration interval:
viewjob.ReseedIntervalColon=Reseed interval:
viewjob.NoScheduledRunTimes=No scheduled run times
Modified: manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties?rev=1555583&r1=1555582&r2=1555583&view=diff
==============================================================================
--- manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties (original)
+++ manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties Sun Jan 5 17:08:27 2014
@@ -368,6 +368,7 @@ editjob.ScheduleTypeColon=ã¹ã±ã
editjob.RescanDocumentsDynamically=åçã«ã³ã³ãã³ããåã¹ãã£ã³
editjob.ScanEveryDocumentOnce=ãã¹ã¦ã®ã³ã³ãã³ããï¼åã¹ãã£ã³
editjob.RecrawlIntervalIfContinuousColon=åèªè¾¼ã¿éé ï¼ç¶ç¶ã®å ´åï¼ï¼
+editjob.MaxRecrawlIntervalIfContinuousColon=Maximum recrawl interval (if continuous):
editjob.minutesBlankInfinity=å ï¼ç©ºç½=ç¡éï¼
editjob.ExpirationIntervalIfContinuousColon=失å¹éé ï¼ç¶ç¶ã®å ´åï¼ï¼
editjob.ReseedIntervalIfContinuousColon=åã·ã¼ãéé ï¼ç¶ç¶ã®å ´åï¼ï¼
@@ -415,6 +416,8 @@ editjob.AddNewScheduleRecord=ã¹ã�
editjob.JobMustHaveAName=ã¸ã§ãåãå
¥åãã¦ãã ãã
editjob.DurationMustBeAValidInteger=æéã«ã¯æ´æ°ãå
¥åãã¦ãã ãã
editjob.RecrawlIntervalMustBeAValidIntegerOrNull=åã¯ãã¼ã«ééã«ã¯æ´æ°åã¯nullãå
¥åãã¦ãã ãã
+editjob.MaxRecrawlIntervalMustBeAValidIntegerOrNull=Maximum recrawl interval must be a valid integer or null
+editjob.MaxRecrawlIntervalMustBeLargerThanRecrawlInterval=Maximum recrawl interval must be larger that recrawl interval
editjob.ReseedIntervalMustBeAValidIntegerOrNull=åã·ã¼ãééã«ã¯æ´æ°åã¯nullãå
¥åãã¦ãã ãã
editjob.ExpirationIntervalMustBeAValidIntegerOrNull=失å¹æéã«ã¯æ´æ°åã¯nullãå
¥åãã¦ãã ãã
editjob.RemoveScheduleRecord=ã¹ã±ã¸ã¥ã¼ã«ã¬ã³ã¼ããåé¤ï¼ #
@@ -774,6 +777,7 @@ viewjob.ApacheManifoldCFViewJob=Apache M
viewjob.ViewAJob=ã¸ã§ãæ
å ±
viewjob.ScheduleTypeColon=ã¹ã±ã¸ã¥ã¼ã«ã¿ã¤ãï¼
viewjob.MinimumRecrawlIntervalColon=æå°åèªè¾¼ã¿ééï¼
+viewjob.MaximumRecrawlIntervalColon=Maximum recrawl interval:
viewjob.ExpirationIntervalColon=失å¹ééï¼
viewjob.ReseedIntervalColon=åã·ã¼ãééï¼
viewjob.NoScheduledRunTimes=ã¹ã±ã¸ã¥ã¼ã«ãããå®è¡æéãããã¾ãã